/// <summary>
        /// Add a decorator to the enumerable for additional processing
        /// </summary>
        /// <param name="operation">The operation.</param>
        /// <param name="enumerator">The enumerator.</param>
        protected override IEnumerable <Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable <Row> enumerator)
        {
            ThreadSafeEnumerator <Row> threadedEnumerator = new ThreadSafeEnumerator <Row>();

            ThreadPool.QueueUserWorkItem(delegate
            {
                try
                {
                    foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
                    {
                        threadedEnumerator.AddItem(t);
                    }
                }
                catch (Exception e)
                {
                    Error(e, "Failed to execute operation {0}", operation);
                    threadedEnumerator.MarkAsFinished();
#if DEBUG
                    throw e;
#endif
                }
                finally
                {
                    threadedEnumerator.MarkAsFinished();
                }
            });
            return(threadedEnumerator);
        }
Exemple #2
0
        /// <summary>
        /// Add a decorator to the enumerable for additional processing
        /// </summary>
        /// <param name="operation">The operation.</param>
        /// <param name="enumerator">The enumerator.</param>
        /// <param name="cancellationToken">A <see cref="T:System.Threading.CancellationToken" /> that may be used to cancel the asynchronous iteration.</param>
        protected override AsyncEnumerableTask <Row> DecorateEnumerableForExecution(
            IOperation operation,
            IAsyncEnumerable <Row> enumerator,
            CancellationToken cancellationToken = default)
        {
            ThreadSafeEnumerator <Row> threadedEnumerator = new ThreadSafeEnumerator <Row>();
            Task task = Task.Run(async() =>
            {
                try
                {
                    IAsyncEnumerable <Row> eventRaisingEnumerator = new EventRaisingEnumerator(operation, enumerator);
                    await eventRaisingEnumerator
                    .ForEachAsync(async t => { await threadedEnumerator.AddItem(t); },
                                  cancellationToken: cancellationToken);
                }
                catch (Exception e)
                {
                    Error(e, "Failed to execute operation {0}", new Tuple <string, object>("Operation", operation));
                }
                finally
                {
                    await threadedEnumerator.MarkAsFinished();
                }
            }, cancellationToken);

            return(new AsyncEnumerableTask <Row>
            {
                Enumerable = threadedEnumerator,
                Task = task,
            });
        }
 public void CanEnumerateSafely()
 {
     var strings = LangTestHelpers.RandomStrings(1000, 50);
     var results = new ConcurrentQueue<string>();
     using (var stringsEnumerator = strings.GetEnumerator())
     {
         var tse = new ThreadSafeEnumerator<string>(stringsEnumerator);
         var threads = new List<Thread>();
         for (var i = 0; i < 10; i++)
         {
             var thread = new Thread(() =>
             {
                 string it = null;
                 while (tse.TryGetNext(ref it))
                 {
                     results.Enqueue(it);
                 }
             });
             thread.Start();
             threads.Add(thread);
         }
         foreach (var thread in threads)
         {
             thread.Join();
         }
     }
     CollectionAssert.AreEquivalent(strings, results);
 }
Exemple #4
0
        public void CanEnumerateSafely()
        {
            var strings = LangTestHelpers.RandomStrings(1000, 50);
            var results = new ConcurrentQueue <string>();

            using (var stringsEnumerator = strings.GetEnumerator())
            {
                var tse     = new ThreadSafeEnumerator <string>(stringsEnumerator);
                var threads = new List <Thread>();
                for (int i = 0; i < 10; i++)
                {
                    var thread = new Thread(() =>
                    {
                        string it = null;
                        while (tse.TryGetNext(ref it))
                        {
                            results.Enqueue(it);
                        }
                    });
                    thread.Start();
                    threads.Add(thread);
                }
                foreach (var thread in threads)
                {
                    thread.Join();
                }
            }
            CollectionAssert.AreEquivalent(strings, results);
        }
 /// <summary>
 /// Add a decorator to the enumerable for additional processing
 /// </summary>
 /// <param name="operation">The operation.</param>
 /// <param name="enumerator">The enumerator.</param>
 protected override IEnumerable<Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable<Row> enumerator)
 {
     ThreadSafeEnumerator<Row> threadedEnumerator = new ThreadSafeEnumerator<Row>();
     ThreadPool.QueueUserWorkItem(delegate
     {
         try
         {
             foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
             {
                 threadedEnumerator.AddItem(t);
             }
         }
         catch (Exception e)
         {
             Error(e, "Failed to execute operation {0}", operation);
             threadedEnumerator.MarkAsFinished();
             throw;
         }
         finally
         {
             threadedEnumerator.MarkAsFinished();
         }
     });
     return threadedEnumerator;
 }
Exemple #6
0
        /// <summary>
        ///     Add a decorator to the enumerable for additional processing
        /// </summary>
        /// <param name="operation">The operation.</param>
        /// <param name="enumerator">The enumerator.</param>
        protected override IEnumerable <Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable <Row> enumerator)
        {
            var threadedEnumerator = new ThreadSafeEnumerator <Row>();

            ThreadPool.QueueUserWorkItem(delegate {
                try {
                    foreach (Row t in new EventRaisingEnumerator(operation, enumerator))
                    {
                        threadedEnumerator.AddItem(t);
                    }
                } catch (Exception e) {
                    foreach (var inner in e.FlattenHierarchy())
                    {
                        Error(inner, "Failed to execute {0}. {1}", operation.Name, inner.Message);
                        Debug(inner.StackTrace);
                    }
                    threadedEnumerator.MarkAsFinished();
#if DEBUG
                    throw new TransformalizeException(Logger, e.Message);
#endif
                } finally {
                    threadedEnumerator.MarkAsFinished();
                }
            });
            return(threadedEnumerator);
        }
Exemple #7
0
        public virtual IEnumerable <TOutput> Execute(IEnumerable <TInput> input)
        {
            var threadSafe = new ThreadSafeEnumerator <TInput>();
            var wrapper    = new ChainedEnumeratorWrapper <TInput>(input, threadSafe);

            return(operation.Execute(wrapper));
        }
        /// <summary>
        ///     Add a decorator to the enumerable for additional processing
        /// </summary>
        /// <param name="operation">The operation.</param>
        /// <param name="enumerator">The enumerator.</param>
        protected override IEnumerable<Row> DecorateEnumerableForExecution(IOperation operation, IEnumerable<Row> enumerator) {
            var threadedEnumerator = new ThreadSafeEnumerator<Row>();
            ThreadPool.QueueUserWorkItem(delegate {
                try {
                    foreach (Row t in new EventRaisingEnumerator(operation, enumerator)) {
                        threadedEnumerator.AddItem(t);
                    }
                } catch (Exception e) {
                    foreach (var inner in e.FlattenHierarchy()) {
                        Error(inner, "Failed to execute {0}. {1}", operation.Name, inner.Message);
                        Debug(inner.StackTrace);
                    }
                    threadedEnumerator.MarkAsFinished();
#if DEBUG
                    throw new TransformalizeException(Logger, e.Message);
#endif

                } finally {
                    threadedEnumerator.MarkAsFinished();
                }
            });
            return threadedEnumerator;
        }
        public void TestGetNextTransferItemsRetryGetChunks2()
        {
            var initialJobResponse = JobResponseStubs.BuildJobResponse(
                JobResponseStubs.Chunk1(JobResponseStubs.NodeId1, false, false),
                JobResponseStubs.Chunk2(JobResponseStubs.NodeId1, false, false),
                JobResponseStubs.Chunk3(JobResponseStubs.NodeId1, false, false)
            );

            var node1Client = new Mock<IDs3Client>(MockBehavior.Strict);

            var factory = new Mock<IDs3ClientFactory>(MockBehavior.Strict);

            factory.Setup(cf => cf.GetClientForNodeId(JobResponseStubs.NodeId1)).Returns(node1Client.Object);

            var client = new Mock<IDs3Client>(MockBehavior.Strict);
            client
                .Setup(c => c.BuildFactory(It.IsAny<IEnumerable<JobNode>>()))
                .Returns(factory.Object);

            var first = true;
            client
                .Setup(
                    c =>
                        c.GetJobChunksReadyForClientProcessingSpectraS3(
                            AllocateMock.AvailableChunks(JobResponseStubs.JobId)))
                .Returns(() =>
                {
                    if (first)
                    {
                        first = false;
                        return GetJobChunksReadyForClientProcessingSpectraS3Response.Success(TimeSpan.FromMinutes(5),
                            JobResponseStubs.BuildJobResponse(JobResponseStubs.Chunk1(JobResponseStubs.NodeId1, true,
                                true)));
                    }

                    return GetJobChunksReadyForClientProcessingSpectraS3Response.Success(TimeSpan.FromMinutes(5),
                        JobResponseStubs.BuildJobResponse(JobResponseStubs.Chunk2(JobResponseStubs.NodeId1, true, true)));
                });

            var source = new ReadStreamChunkStrategy(_ => { }, 5);

            using (var transfers = source.GetNextTransferItems(client.Object, initialJobResponse).GetEnumerator())
            {
                var itemGetter = new ThreadSafeEnumerator<TransferItem>(transfers);
                Assert.Throws<Ds3NoMoreRetriesException>(() =>
                {
                    TransferItem it = null;
                    itemGetter.TryGetNext(ref it); //the first blob in the first chunk will return
                    source.CompleteBlob(it.Blob);

                    itemGetter.TryGetNext(ref it); //the second blob in the first chunk will return
                    source.CompleteBlob(it.Blob);

                    itemGetter.TryGetNext(ref it); //the first blob in the second chunk will return
                    source.CompleteBlob(it.Blob);

                    itemGetter.TryGetNext(ref it); //the second blob in the second chunk will return
                    source.CompleteBlob(it.Blob);

                    itemGetter.TryGetNext(ref it);
                        //exception will be thrown since we will get the same chunk over and over
                });
            }

            node1Client.VerifyAll();
            factory.VerifyAll();
        }