public void ForEachTest() { Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF)); ExceptionCollector ec = new ExceptionCollector(); AsyncQueue <int> q1 = new AsyncQueue <int>(5); Func <Task> t1 = async delegate() { for (int i = 0; i < 100; ++i) { await q1.Enqueue(r.Next(1000), ec.CancellationToken); } q1.WriteEof(); }; AsyncQueue <int> q2 = new AsyncQueue <int>(5); Func <Task> t2 = WorkerTask.ForEach ( q1, async delegate(ForEachInfo <int> fi) { await q2.Enqueue(fi.Item * 100, fi.CancellationToken); }, delegate() { q2.WriteEof(); return(Task.CompletedTask); }, ec ); List <int> items = new List <int>(); Func <Task> t3 = WorkerTask.ForEach ( q2, delegate(ForEachInfo <int> fi) { items.Add(fi.Item); return(Task.CompletedTask); }, delegate() { return(Task.CompletedTask); }, ec ); Task T1 = Task.Run(t1); Task T2 = Task.Run(t2); Task T3 = Task.Run(t3); Task.WaitAll(T1, T2, T3); Assert.AreEqual(0, ec.Exceptions.Count); Assert.AreEqual(100, items.Count); }
public static IQueueSource <U> Select <T, U>(this IQueueSource <T> queue, Func <T, Task <U> > func, int?capacity = null) { AsyncQueue <U> outQueue = new AsyncQueue <U>(capacity ?? 2); Func <Task> worker = async delegate() { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } U item2 = await func(item.Value); await outQueue.Enqueue(item2, CancellationToken.None); } outQueue.WriteEof(); }; Task.Run(worker); return(outQueue); }
public static IQueueSource <T> Where <T>(this IQueueSource <T> queue, Func <T, Task <bool> > predicate, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); Func <Task> worker = async delegate() { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } if (await predicate(item.Value)) { await outQueue.Enqueue(item.Value, CancellationToken.None); } } outQueue.WriteEof(); }; Task _dummy = Task.Run(worker); return(outQueue); }
private async Task PutItems(string prefix, AsyncQueue <string> dest, List <Item> items) { foreach (Item i in items) { await Task.Delay(i.delay); await dest.Enqueue(prefix + i.value, CancellationToken.None); } dest.WriteEof(); }
private async Task ExtMethodQueueTestAsync() { AsyncQueue <string> q = new AsyncQueue <string>(5); Func <Task> producer = async delegate() { #region for (int i = 0; i < 20; ++i) { System.Diagnostics.Debug.WriteLine($"Writing {i}..."); await q.Enqueue($"Value: {i}", CancellationToken.None); } System.Diagnostics.Debug.WriteLine("Writing EOF..."); q.WriteEof(); #endregion }; Func <Task> consumer = async delegate() { #region bool more = true; while (more) { System.Diagnostics.Debug.WriteLine("Reading..."); Option <string> ostr = await q.Dequeue(CancellationToken.None); if (ostr.HasValue) { System.Diagnostics.Debug.WriteLine($"Read {ostr.Value}"); } else { System.Diagnostics.Debug.WriteLine("Read EOF..."); more = false; } } #endregion }; Task tProducer = Task.Run(producer); Task tConsumer = Task.Run(consumer); await Task.WhenAll(tProducer, tConsumer); }
public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); IdleDetector idleDetector = new IdleDetector(); Func <Task> workPoster = async delegate() { while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } T itemValue = item.Value; idleDetector.Enter(); Task doingWork = await parallelWorker.StartWorkItem ( async (int workerId) => { try { if (await predicate(itemValue)) { await outQueue.Enqueue(itemValue, CancellationToken.None); } } finally { idleDetector.Leave(); } }, CancellationToken.None ); } await idleDetector.WaitForIdle(CancellationToken.None); outQueue.WriteEof(); }; Task.Run(workPoster); return(outQueue); }
public static IQueueSource <T> AsQueueSource <T>(this IEnumerable <T> items, int?capacity = null) { AsyncQueue <T> queue = new AsyncQueue <T>(capacity ?? 2); Func <Task> feed = async delegate() { foreach (var item in items) { await queue.Enqueue(item, CancellationToken.None); } queue.WriteEof(); }; Task _dummy = Task.Run(feed); return(queue); }
public static IQueueSource <T> Reorder <T>(this IQueueSource <T> queue, Func <T, int> getOrder, int first, int?capacity = null) { AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2); Func <Task> worker = async delegate() { int next = first; ImmutableDictionary <int, T> buffer = ImmutableDictionary <int, T> .Empty; while (true) { Option <T> item = await queue.Dequeue(CancellationToken.None); if (!item.HasValue) { break; } int order = getOrder(item.Value); if (order == next) { await outQueue.Enqueue(item.Value, CancellationToken.None); ++next; while (buffer.ContainsKey(next)) { await outQueue.Enqueue(buffer[next], CancellationToken.None); buffer = buffer.Remove(next); ++next; } } else { buffer = buffer.Add(order, item.Value); } } outQueue.WriteEof(); }; Task.Run(worker); return(outQueue); }
public void MultiSourceForEachExceptionTest() { ExceptionCollector ec = new ExceptionCollector(); AsyncQueue <int> q1 = new AsyncQueue <int>(5); Func <Task> t1 = async delegate() { for (int i = 0; i < 100; ++i) { await q1.Enqueue(i, ec.CancellationToken); } q1.WriteEof(); }; AsyncQueue <int> q2 = new AsyncQueue <int>(5); Func <Task> t2 = async delegate() { for (int i = 100; i < 200; ++i) { await q2.Enqueue(i, ec.CancellationToken); } q2.WriteEof(); }; AsyncQueue <int> q3 = new AsyncQueue <int>(5); Func <Task> t3 = WorkerTask.ForEach ( new IQueueSource <int>[] { q1, q2 }, InputPriorities.RoundRobin, async fei => { if (fei.Item == 133) { throw new InvalidOperationException("test"); } await q3.Enqueue(fei.Item, fei.CancellationToken); }, () => { q3.WriteEof(); return(Task.CompletedTask); }, ec ); Func <Task> t4 = WorkerTask.ForEach ( q3, fei => { System.Diagnostics.Debug.WriteLine(fei.Item); return(Task.CompletedTask); }, null, ec ); Task T1 = Task.Run(t1); Task T2 = Task.Run(t2); Task T3 = Task.Run(t3); Task T4 = Task.Run(t4); try { ec.WaitAll(T1, T2, T3, T4); } catch (Exception exc) { System.Diagnostics.Debug.WriteLine(exc); } }
public void ParallelForEachExceptionTest() { ExceptionCollector ec = new ExceptionCollector(); AsyncQueue <int> q1 = new AsyncQueue <int>(5); Func <Task> t1 = async delegate() { for (int i = 0; i < 100; ++i) { await q1.Enqueue(i, ec.CancellationToken); } q1.WriteEof(); }; AsyncQueue <int> q2 = new AsyncQueue <int>(5); Func <Task> t2 = WorkerTask.ForEach ( q1, async delegate(ForEachInfo <int> fi) { await q2.Enqueue(fi.Item * 100, fi.CancellationToken); if (fi.Item == 53) { throw new InvalidOperationException("test 1"); } }, delegate() { q2.WriteEof(); return(Task.CompletedTask); }, ec ); List <int> items = new List <int>(); Func <Task> t3 = WorkerTask.ForEach ( q2, delegate(ForEachInfo <int> fi) { items.Add(fi.Item); return(Task.CompletedTask); }, delegate() { return(Task.CompletedTask); }, ec ); Task T1 = Task.Run(t1); Task T2 = Task.Run(t2); Task T3 = Task.Run(t3); bool threwException = false; try { ec.WaitAll(T1, T2, T3); } catch (Exception exc) { System.Diagnostics.Debug.WriteLine(exc); threwException = true; } Assert.IsTrue(threwException); }
private async Task CompleteAnyWithCancellationAsync() { AsyncQueue <string> q1 = new AsyncQueue <string>(5); AsyncQueue <string> q2 = new AsyncQueue <string>(5); Func <Task> producer1 = async delegate() { await Task.Delay(2500); await q1.Enqueue("one", CancellationToken.None); q1.WriteEof(); }; Func <Task> producer2 = async delegate() { await Task.Delay(5500); await q2.Enqueue("two", CancellationToken.None); q2.WriteEof(); }; Func <Task> consumer = async delegate() { bool q1eof = false; bool q2eof = false; while (!q1eof || !q2eof) { Tuple <int, string> r1 = null; using (CancellationTokenSource s1 = new CancellationTokenSource(1000)) { try { r1 = await Utils.OperationStarters <int, string>() .AddIf(!q1eof, 1, Utils.StartableGet(q1, x => x, null)) .AddIf(!q2eof, 2, Utils.StartableGet(q2, x => x, null)) .CompleteAny(s1.Token); } catch (OperationCanceledException) { } } if (r1 == null) { System.Diagnostics.Debug.WriteLine("Cancelled (because of timeout)"); } else if (r1.Item1 == 1) { if (r1.Item2 == null) { System.Diagnostics.Debug.WriteLine("Got EOF from 1"); q1eof = true; } else { System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 1"); } } else { Assert.AreEqual(2, r1.Item1); if (r1.Item2 == null) { System.Diagnostics.Debug.WriteLine("Got EOF from 2"); q2eof = true; } else { System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 2"); } } } }; Task p1 = Task.Run(producer1); Task p2 = Task.Run(producer2); Task c = Task.Run(consumer); await Task.WhenAll(p1, p2, c); }
private async Task CompleteAnyWithCancellationAsync() { AsyncQueue<string> q1 = new AsyncQueue<string>(5); AsyncQueue<string> q2 = new AsyncQueue<string>(5); Func<Task> producer1 = async delegate () { await Task.Delay(2500); await q1.Enqueue("one", CancellationToken.None); q1.WriteEof(); }; Func<Task> producer2 = async delegate () { await Task.Delay(5500); await q2.Enqueue("two", CancellationToken.None); q2.WriteEof(); }; Func<Task> consumer = async delegate () { bool q1eof = false; bool q2eof = false; while (!q1eof || !q2eof) { Tuple<int, string> r1 = null; using (CancellationTokenSource s1 = new CancellationTokenSource(1000)) { try { r1 = await Utils.OperationStarters<int, string>() .AddIf(!q1eof, 1, Utils.StartableGet(q1, x => x, null)) .AddIf(!q2eof, 2, Utils.StartableGet(q2, x => x, null)) .CompleteAny(s1.Token); } catch (OperationCanceledException) { } } if (r1 == null) { System.Diagnostics.Debug.WriteLine("Cancelled (because of timeout)"); } else if (r1.Item1 == 1) { if (r1.Item2 == null) { System.Diagnostics.Debug.WriteLine("Got EOF from 1"); q1eof = true; } else { System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 1"); } } else { Assert.AreEqual(2, r1.Item1); if (r1.Item2 == null) { System.Diagnostics.Debug.WriteLine("Got EOF from 2"); q2eof = true; } else { System.Diagnostics.Debug.WriteLine($"Got {r1.Item2} from 2"); } } } }; Task p1 = Task.Run(producer1); Task p2 = Task.Run(producer2); Task c = Task.Run(consumer); await Task.WhenAll(p1, p2, c); }
private async Task PutItems(string prefix, AsyncQueue<string> dest, List<Item> items) { foreach(Item i in items) { await Task.Delay(i.delay); await dest.Enqueue(prefix + i.value, CancellationToken.None); } dest.WriteEof(); }
private async Task ExtMethodQueueTestAsync() { AsyncQueue<string> q = new AsyncQueue<string>(5); Func<Task> producer = async delegate () { #region for (int i = 0; i < 20; ++i) { System.Diagnostics.Debug.WriteLine($"Writing {i}..."); await q.Enqueue($"Value: {i}", CancellationToken.None); } System.Diagnostics.Debug.WriteLine("Writing EOF..."); q.WriteEof(); #endregion }; Func<Task> consumer = async delegate () { #region bool more = true; while (more) { System.Diagnostics.Debug.WriteLine("Reading..."); Option<string> ostr = await q.Dequeue(CancellationToken.None); if (ostr.HasValue) { System.Diagnostics.Debug.WriteLine($"Read {ostr.Value}"); } else { System.Diagnostics.Debug.WriteLine("Read EOF..."); more = false; } } #endregion }; Task tProducer = Task.Run(producer); Task tConsumer = Task.Run(consumer); await Task.WhenAll(tProducer, tConsumer); }
private async Task SimpleQueueTestAsync() { AsyncQueue<int> queue = new AsyncQueue<int>(11); Func<Task> producer = async delegate() { #region for (int i = 0; i < 20; ++i) { System.Diagnostics.Debug.WriteLine("Acquiring write..."); AcquireWriteResult result = await queue.AcquireWriteAsync(1, CancellationToken.None); result.Visit<DBNull> ( new Func<AcquireWriteSucceeded, DBNull> ( succeeded => { System.Diagnostics.Debug.WriteLine("Write-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " spaces"); Assert.AreEqual(1, succeeded.ItemCount); if (succeeded.ItemCount >= 1) { System.Diagnostics.Debug.WriteLine("Releasing write (1)..."); queue.ReleaseWrite(i); } else { System.Diagnostics.Debug.WriteLine("Releasing write (0)..."); queue.ReleaseWrite(); } return DBNull.Value; } ), new Func<AcquireWriteCancelled, DBNull> ( cancelled => { throw new OperationCanceledException(); } ), new Func<AcquireWriteFaulted, DBNull> ( faulted => { throw faulted.Exception; } ) ); } System.Diagnostics.Debug.WriteLine("Writing EOF..."); queue.WriteEof(); #endregion }; Func<Task> consumer = async delegate() { #region bool more = true; while (more) { System.Diagnostics.Debug.WriteLine("Acquiring read..."); const int ACQUIRE_COUNT = 3; AcquireReadResult result = await queue.AcquireReadAsync(ACQUIRE_COUNT, CancellationToken.None); result.Visit<DBNull> ( new Func<AcquireReadSucceeded, DBNull> ( succeeded => { System.Diagnostics.Debug.WriteLine("Read-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " items"); Assert.IsInstanceOfType(succeeded, typeof(AcquireReadSucceeded<int>)); if (succeeded is AcquireReadSucceeded<int>) { AcquireReadSucceeded<int> succeeded2 = (AcquireReadSucceeded<int>)succeeded; System.Diagnostics.Debug.WriteLine("{ " + string.Join(", ", succeeded2.Items) + " }"); } if (succeeded.ItemCount < ACQUIRE_COUNT) { System.Diagnostics.Debug.WriteLine("Setting \"more\" flag to false..."); more = false; } System.Diagnostics.Debug.WriteLine("Releasing read (" + succeeded.ItemCount + ")..."); queue.ReleaseRead(succeeded.ItemCount); return DBNull.Value; } ), new Func<AcquireReadCancelled, DBNull> ( cancelled => { throw new OperationCanceledException(); } ), new Func<AcquireReadFaulted, DBNull> ( faulted => { throw faulted.Exception; } ) ); } #endregion }; Task tProducer = Task.Run(producer); Task tConsumer = Task.Run(consumer); await Task.WhenAll(tProducer, tConsumer); }
public void ParallelMultiSourceForEachExceptionTest() { object rSyncRoot = new object(); Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF)); ExceptionCollector ec = new ExceptionCollector(); AsyncQueue <int> q1 = new AsyncQueue <int>(5); Func <Task> t1 = async delegate() { for (int i = 0; i < 100; ++i) { await q1.Enqueue(i, ec.CancellationToken); } q1.WriteEof(); }; AsyncQueue <int> q2 = new AsyncQueue <int>(5); Func <Task> t2 = async delegate() { for (int i = 100; i < 200; ++i) { await q2.Enqueue(i, ec.CancellationToken); } q2.WriteEof(); }; AsyncQueue <int> q3 = new AsyncQueue <int>(5); ParallelWorker pw = new ParallelWorker(8); Func <Task> t3 = WorkerTask.ParallelForEach ( new IQueueSource <int>[] { q1, q2 }, InputPriorities.RoundRobin, pw, async fei => { await q3.Enqueue(fei.Item, fei.CancellationToken); if (fei.Item == 133) { throw new InvalidOperationException("test"); } int time; lock (rSyncRoot) { time = 100 + r.Next(800); } await Task.Delay(time); }, () => { q3.WriteEof(); return(Task.CompletedTask); }, ec ); Func <Task> t4 = WorkerTask.ForEach ( q3, fei => { System.Diagnostics.Debug.WriteLine(fei.Item); return(Task.CompletedTask); }, null, ec ); Task T1 = Task.Run(t1); Task T2 = Task.Run(t2); Task T3 = Task.Run(t3); Task T4 = Task.Run(t4); try { ec.WaitAll(T1, T2, T3, T4); } catch (Exception exc) { System.Diagnostics.Debug.WriteLine(exc); } }
public void ForEachWithExceptionTest() { Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF)); ExceptionCollector ec = new ExceptionCollector(); AsyncQueue <int> q1 = new AsyncQueue <int>(5); Func <Task> t1 = async delegate() { for (int i = 0; i < 100; ++i) { await q1.Enqueue(r.Next(1000), ec.CancellationToken); } q1.WriteEof(); }; AsyncQueue <int> q2 = new AsyncQueue <int>(5); Func <Task> t2 = WorkerTask.ForEach ( q1, async delegate(ForEachInfo <int> fi) { await q2.Enqueue(fi.Item * 100, fi.CancellationToken); throw new InvalidOperationException("test 1"); }, delegate() { q2.WriteEof(); return(Task.FromException(new InvalidOperationException("test 2"))); }, ec ); List <int> items = new List <int>(); Func <Task> t3 = WorkerTask.ForEach ( q2, delegate(ForEachInfo <int> fi) { items.Add(fi.Item); return(Task.CompletedTask); }, delegate() { return(Task.CompletedTask); }, ec ); Task T1 = Task.Run(t1); Task T2 = Task.Run(t2); Task T3 = Task.Run(t3); bool threwException = false; try { ec.WaitAll(T1, T2, T3); } catch (Exception exc) { System.Diagnostics.Debug.WriteLine(exc); threwException = true; } Assert.IsTrue(threwException); }
private async Task SimpleQueueTestAsync() { AsyncQueue <int> queue = new AsyncQueue <int>(11); Func <Task> producer = async delegate() { #region for (int i = 0; i < 20; ++i) { System.Diagnostics.Debug.WriteLine("Acquiring write..."); AcquireWriteResult result = await queue.AcquireWriteAsync(1, CancellationToken.None); result.Visit <DBNull> ( new Func <AcquireWriteSucceeded, DBNull> ( succeeded => { System.Diagnostics.Debug.WriteLine("Write-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " spaces"); Assert.AreEqual(1, succeeded.ItemCount); if (succeeded.ItemCount >= 1) { System.Diagnostics.Debug.WriteLine("Releasing write (1)..."); queue.ReleaseWrite(i); } else { System.Diagnostics.Debug.WriteLine("Releasing write (0)..."); queue.ReleaseWrite(); } return(DBNull.Value); } ), new Func <AcquireWriteCancelled, DBNull> ( cancelled => { throw new OperationCanceledException(); } ), new Func <AcquireWriteFaulted, DBNull> ( faulted => { throw faulted.Exception; } ) ); } System.Diagnostics.Debug.WriteLine("Writing EOF..."); queue.WriteEof(); #endregion }; Func <Task> consumer = async delegate() { #region bool more = true; while (more) { System.Diagnostics.Debug.WriteLine("Acquiring read..."); const int ACQUIRE_COUNT = 3; AcquireReadResult result = await queue.AcquireReadAsync(ACQUIRE_COUNT, CancellationToken.None); result.Visit <DBNull> ( new Func <AcquireReadSucceeded, DBNull> ( succeeded => { System.Diagnostics.Debug.WriteLine("Read-acquire succeeded, offset " + succeeded.Offset + ", acquired " + succeeded.ItemCount + " items"); Assert.IsInstanceOfType(succeeded, typeof(AcquireReadSucceeded <int>)); if (succeeded is AcquireReadSucceeded <int> ) { AcquireReadSucceeded <int> succeeded2 = (AcquireReadSucceeded <int>)succeeded; System.Diagnostics.Debug.WriteLine("{ " + string.Join(", ", succeeded2.Items) + " }"); } if (succeeded.ItemCount < ACQUIRE_COUNT) { System.Diagnostics.Debug.WriteLine("Setting \"more\" flag to false..."); more = false; } System.Diagnostics.Debug.WriteLine("Releasing read (" + succeeded.ItemCount + ")..."); queue.ReleaseRead(succeeded.ItemCount); return(DBNull.Value); } ), new Func <AcquireReadCancelled, DBNull> ( cancelled => { throw new OperationCanceledException(); } ), new Func <AcquireReadFaulted, DBNull> ( faulted => { throw faulted.Exception; } ) ); } #endregion }; Task tProducer = Task.Run(producer); Task tConsumer = Task.Run(consumer); await Task.WhenAll(tProducer, tConsumer); }