示例#1
0
 /// <summary>
 /// When some specified chunk size is reached, combiners marks the functionality end on that chunk and send it to Reducer for further processing.
 /// And resets its internal state for next chunk.
 /// </summary>
 /// <returns>Sends the chunk to Reducer.</returns>
 public object FinishChunk()
 {
     lock (mutex)
     {
         return(_aggregator.Aggregate(_combinerList));
     }
 }
        public async Task <List <TRslt> > DoAsync(IEnumerable <AbstractRequestInstructions <TRslt, TReq, TRspns> > requests, TimeSpan gatherTimeout, IAggregator <TRslt> gatherer)
        {
            var requestsList = requests?.ToList();

            if (requestsList == null || requestsList.Count <= 0 || gatherTimeout == TimeSpan.MinValue ||
                gatherTimeout == null)
            {
                throw new ArgumentNullException($"{nameof(requests)} or {nameof(gatherTimeout)} or {nameof(gatherer)}");
            }

            var taskBag     = new List <Task>();
            var listResults = new List <ResultEnvelope <TRslt> >();
            var cts         = new CancellationTokenSource();

            cts.CancelAfter(gatherTimeout);
            foreach (var request in requestsList)
            {
                var client = request.ClientManager.GetClient(request.Request.Host);
                if (client != null)
                {
                    taskBag.Add(client.SendAsync(request.Request, cts.Token)
                                .ContinueWith(x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
                }
            }
            var nowPlusGatherTimeout = DateTime.Now.Add(gatherTimeout);

            while (!cts.Token.IsCancellationRequested || nowPlusGatherTimeout < DateTime.Now)
            {
                await Task.Delay(100, cts.Token);
            }
            var result = gatherer.Aggregate(listResults);

            return(result.Select(x => x.Result).ToList());
        }
示例#3
0
        public void AggregatorStrategy(
            IEvaluationEngine engine,
            IAggregator <int, int, Missing> aggregator,
            int answer)
        {
            "establish an evaluatione engine"._(() =>
            {
                engine = new EvaluationEngine();

                aggregator = A.Fake <IAggregator <int, int, Missing> >();
                A.CallTo(() => aggregator.Aggregate(A <IEnumerable <IExpression <int, Missing> > > ._, Missing.Value, A <Context> ._)).Returns(TheAnswer);
            });

            "when defining aggregator strategy"._(() =>
            {
                engine.Solve <HowManyFruitsAreThere, int>()
                .WithAggregatorStrategy()
                .AggregateWith(aggregator);

                answer = engine.Answer(new HowManyFruitsAreThere());
            });

            "it should use aggregator strategy to answer the question"._(() =>
            {
                answer.Should().Be(TheAnswer);
            });
        }
        public async Task <List <TRslt> > DoAsync(AbstractRequestInstructions <TRslt, TReq, TRspns> request,
                                                  List <IHost> hosts, IAggregator <TRslt> gatherer)
        {
            if (request.ClientManager == null)
            {
                throw new ArgumentNullException($"{nameof(request.ClientManager)}");
            }
            var listResults = new List <ResultEnvelope <TRslt> >();
            var taskBag     = new List <Task>();
            var cts         = new CancellationTokenSource();

            cts.CancelAfter(request.Timeout);
            foreach (var host in hosts)
            {
                var client = request.ClientManager.GetClient(host);
                if (client != null)
                {
                    taskBag.Add(client.SendAsync(request.Request, cts.Token)
                                .ContinueWith(x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
                }
            }
            await Task.WhenAll(taskBag).ConfigureAwait(false);

            var result = gatherer.Aggregate(listResults);

            return(result.Select(x => x.Result).ToList());
        }
示例#5
0
        private void AggregatorBaseBehaviors(IAggregator aggregator, bool requiresColumns = true)
        {
            // Verify ToString returns the aggregator type, which matches the start of the class name
            string name = aggregator.ToString();

            Assert.AreEqual(aggregator.GetType().Name.ToLowerInvariant(), (name + "aggregator").ToLowerInvariant());

            // Verify Merge throws if the values are null
            Verify.Exception <ArgumentNullException>(() => aggregator.Merge(null, null));

            // Verify Aggregate throws if the matches or columns are null
            Verify.Exception <ArgumentNullException>(() => aggregator.Aggregate(null, null, new IUntypedColumn[1] {
                ColumnFactory.Build(new ColumnDetails("ID", "int", null), 100)
            }));

            if (requiresColumns)
            {
                ShortSet sample = new ShortSet(100);
                sample.Or(new ushort[] { 1, 2, 3 });
                Verify.Exception <ArgumentException>(() => aggregator.Aggregate(null, sample, null));
            }
        }
        public static void AddAggregator(IDataPublisher publisher,
                                         IAggregator <double, double> agg,
                                         Action <double, string> act, int symbolId)
        {
            eventAsObservable = from update in  Observable.FromEvent <OnFeedReceived, Feed>(
                mktH => publisher.FeedArrived += mktH,
                mktH => publisher.FeedArrived -= mktH
                ).Where((feed) => feed.SymbolId == symbolId)
                                select update.LTP;

            //aggregates and yields results.
            subs = eventAsObservable
                   .Scan <double, double>(0,
                                          (acc, currentValue) =>
            {
                return(agg.Aggregate(currentValue));
            }).Subscribe((state) => act(state, Constants.REDIS_MVA_ROOM_PREFIX + symbolId));
        }
        public static void AddAggregator(IDataPublisher publisher, 
            IAggregator<double, double> agg, 
            Action<double, string> act, int symbolId)
        {
            eventAsObservable = from update in  Observable.FromEvent<OnFeedReceived, Feed>(
                mktH => publisher.FeedArrived += mktH,
                mktH => publisher.FeedArrived -= mktH
                ).Where((feed) => feed.SymbolId == symbolId)
                                    select update.LTP;

            //aggregates and yields results.
            subs = eventAsObservable
                .Scan<double, double>(0,
                (acc, currentValue) =>
                {
                    return agg.Aggregate(currentValue);
                }).Subscribe((state) => act(state, Constants.REDIS_MVA_ROOM_PREFIX + symbolId));
        }
        public IList <Record> Search(Query query)
        {
            IList <IList <Record> > results = new List <IList <Record> >();

            var tasks = new List <Task <IList <Record> > >();

            Parallel.ForEach(
                _scrapers,
                scraper =>
            {
                try
                {
                    results.Add(scraper.Search(query, PAGES));
                }
                catch (Exception)
                {
                }
            });


            return(_aggregator.Aggregate(results));
        }
        public async Task <List <T> > DoHttpRequestForSeveralRecipientsAsync <T>(HttpRequestInstructions <T> request, List <Uri> hosts, IAggregator <T> gatherer)
        {
            var listResults      = new List <ResultEnvelope <T> >();
            var listTasksToAwait = new List <Task>();
            var cts = new CancellationTokenSource();

            cts.CancelAfter(request.Timeout);
            var testOutput = new StringBuilder();

            foreach (var host in hosts)
            {
                try
                {
                    var client = clientManager.GetClient(new HttpHost(host.OriginalString));
                    listTasksToAwait.Add(client.SendAsync(request.Request.Copy(), cts.Token).ContinueWith(
                                             x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
                    testOutput.AppendLine($"this host worked fine: {host.AbsoluteUri}");
                }
                catch (Exception e)
                {
                    testOutput.AppendLine("foreach Catch->" + e.Message);
                }
            }

            try
            {
                Task.WaitAll(listTasksToAwait.ToArray());
                //await Task.WhenAll(listTasksToAwait).ContinueWith(x => testOutput.AppendLine(x.Status.ToString()), TaskContinuationOptions.OnlyOnFaulted);
            }
            catch (Exception e) { testOutput.AppendLine("outer catch->" + e.Message); }
            testOutput.AppendLine($"listResults.Count = {listResults.Count}; {string.Join(",", listResults.Select(x => x.Result))}");
            listResults = listResults.Where(x => x != null).ToList();
            testOutput.AppendLine($"listResults.Count = {listResults.Count}; {string.Join(",", listResults.Select(x => x.Result))}");
            System.IO.File.WriteAllText(@"C:\users\az185030\desktop\sampletestresults.txt", testOutput.ToString());
            var result = gatherer.Aggregate(listResults);

            return(result.Select(x => x.Result).ToList());
        }
        public async Task <List <T> > DoSeveralHttpRequestsAsync <T>(List <HttpRequestInstructions <T> > requests, TimeSpan gatherTimeout, IAggregator <T> gatherer)
        {
            //TODO null checks all around
            var listResults      = new List <ResultEnvelope <T> >();
            var listTasksToAwait = new List <Task>();
            var cts = new CancellationTokenSource();

            cts.CancelAfter(gatherTimeout);
            foreach (var request in requests)
            {
                var client = clientManager.GetClient(new HttpHost(request.Request.RequestUri.ToString()));
                if (client != null)
                {
                    listTasksToAwait.Add(client.SendAsync(request.Request.Copy(), cts.Token).ContinueWith(
                                             x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
                }
            }
            await Task.WhenAll(listTasksToAwait).ConfigureAwait(false);

            var result = gatherer.Aggregate(listResults);

            return(result.Select(x => x.Result).ToList());
        }
示例#11
0
        private static void AggregateAllDimensionsFlat(object aggregationContext, DataBlock block, ushort itemCount, ShortSet baseWhereSet, List <List <Tuple <IExpression, ShortSet> > > allDimensionValueSets, IUntypedColumn[] columns, IAggregator aggregator)
        {
            int nextBlockRow = 0;

            int dimensionCount = allDimensionValueSets.Count;

            int currentDimension = 0;

            int[]      nextIndexPerDimension = new int[dimensionCount];
            ShortSet[] setsPerDimension      = new ShortSet[dimensionCount + 1];

            setsPerDimension[0] = baseWhereSet;
            for (int i = 1; i < setsPerDimension.Length; ++i)
            {
                setsPerDimension[i] = new ShortSet(itemCount);
            }

            object[] aggregationResults = new object[block.RowCount];

            while (currentDimension >= 0)
            {
                if (currentDimension == dimensionCount)
                {
                    // Leaf: Compute Aggregate, then ask for next combination
                    aggregationResults[nextBlockRow] = aggregator.Aggregate(aggregationContext, setsPerDimension[currentDimension], columns);
                    nextBlockRow++;
                    currentDimension--;
                }
                else if (nextIndexPerDimension[currentDimension] < allDimensionValueSets[currentDimension].Count)
                {
                    // Non-leaf, more values: Get next value and combine
                    Tuple <IExpression, ShortSet> dimensionValue = allDimensionValueSets[currentDimension][nextIndexPerDimension[currentDimension]];

                    ShortSet current = setsPerDimension[currentDimension + 1];
                    current.FromAnd(setsPerDimension[currentDimension], dimensionValue.Item2);

                    nextIndexPerDimension[currentDimension]++;
                    currentDimension++;

                    // Skip rest of range if set already empty [slower; IsEmpty too expensive]
                    if (current.IsEmpty())
                    {
                        int rowsToSkip = 1;
                        for (int i = currentDimension; i < dimensionCount; ++i)
                        {
                            rowsToSkip *= allDimensionValueSets[i].Count;
                        }

                        nextBlockRow += rowsToSkip;
                        currentDimension--;
                    }
                }
                else
                {
                    // Non-leaf, no more values: Pop up to previous dimension for next value there
                    nextIndexPerDimension[currentDimension] = 0;
                    currentDimension--;
                }
            }

            block.SetColumn(block.ColumnCount - 1, aggregationResults);
        }