/// <summary>
/// When some specified chunk size is reached, combiners marks the functionality end on that chunk and send it to Reducer for further processing.
/// And resets its internal state for next chunk.
/// </summary>
/// <returns>Sends the chunk to Reducer.</returns>
public object FinishChunk()
{
lock (mutex)
{
return(_aggregator.Aggregate(_combinerList));
}
}
public async Task <List <TRslt> > DoAsync(IEnumerable <AbstractRequestInstructions <TRslt, TReq, TRspns> > requests, TimeSpan gatherTimeout, IAggregator <TRslt> gatherer)
{
var requestsList = requests?.ToList();
if (requestsList == null || requestsList.Count <= 0 || gatherTimeout == TimeSpan.MinValue ||
gatherTimeout == null)
{
throw new ArgumentNullException($"{nameof(requests)} or {nameof(gatherTimeout)} or {nameof(gatherer)}");
}
var taskBag = new List <Task>();
var listResults = new List <ResultEnvelope <TRslt> >();
var cts = new CancellationTokenSource();
cts.CancelAfter(gatherTimeout);
foreach (var request in requestsList)
{
var client = request.ClientManager.GetClient(request.Request.Host);
if (client != null)
{
taskBag.Add(client.SendAsync(request.Request, cts.Token)
.ContinueWith(x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
}
}
var nowPlusGatherTimeout = DateTime.Now.Add(gatherTimeout);
while (!cts.Token.IsCancellationRequested || nowPlusGatherTimeout < DateTime.Now)
{
await Task.Delay(100, cts.Token);
}
var result = gatherer.Aggregate(listResults);
return(result.Select(x => x.Result).ToList());
}
public void AggregatorStrategy(
IEvaluationEngine engine,
IAggregator <int, int, Missing> aggregator,
int answer)
{
"establish an evaluatione engine"._(() =>
{
engine = new EvaluationEngine();
aggregator = A.Fake <IAggregator <int, int, Missing> >();
A.CallTo(() => aggregator.Aggregate(A <IEnumerable <IExpression <int, Missing> > > ._, Missing.Value, A <Context> ._)).Returns(TheAnswer);
});
"when defining aggregator strategy"._(() =>
{
engine.Solve <HowManyFruitsAreThere, int>()
.WithAggregatorStrategy()
.AggregateWith(aggregator);
answer = engine.Answer(new HowManyFruitsAreThere());
});
"it should use aggregator strategy to answer the question"._(() =>
{
answer.Should().Be(TheAnswer);
});
}
public async Task <List <TRslt> > DoAsync(AbstractRequestInstructions <TRslt, TReq, TRspns> request,
List <IHost> hosts, IAggregator <TRslt> gatherer)
{
if (request.ClientManager == null)
{
throw new ArgumentNullException($"{nameof(request.ClientManager)}");
}
var listResults = new List <ResultEnvelope <TRslt> >();
var taskBag = new List <Task>();
var cts = new CancellationTokenSource();
cts.CancelAfter(request.Timeout);
foreach (var host in hosts)
{
var client = request.ClientManager.GetClient(host);
if (client != null)
{
taskBag.Add(client.SendAsync(request.Request, cts.Token)
.ContinueWith(x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
}
}
await Task.WhenAll(taskBag).ConfigureAwait(false);
var result = gatherer.Aggregate(listResults);
return(result.Select(x => x.Result).ToList());
}
private void AggregatorBaseBehaviors(IAggregator aggregator, bool requiresColumns = true)
{
// Verify ToString returns the aggregator type, which matches the start of the class name
string name = aggregator.ToString();
Assert.AreEqual(aggregator.GetType().Name.ToLowerInvariant(), (name + "aggregator").ToLowerInvariant());
// Verify Merge throws if the values are null
Verify.Exception <ArgumentNullException>(() => aggregator.Merge(null, null));
// Verify Aggregate throws if the matches or columns are null
Verify.Exception <ArgumentNullException>(() => aggregator.Aggregate(null, null, new IUntypedColumn[1] {
ColumnFactory.Build(new ColumnDetails("ID", "int", null), 100)
}));
if (requiresColumns)
{
ShortSet sample = new ShortSet(100);
sample.Or(new ushort[] { 1, 2, 3 });
Verify.Exception <ArgumentException>(() => aggregator.Aggregate(null, sample, null));
}
}
public static void AddAggregator(IDataPublisher publisher,
IAggregator <double, double> agg,
Action <double, string> act, int symbolId)
{
eventAsObservable = from update in Observable.FromEvent <OnFeedReceived, Feed>(
mktH => publisher.FeedArrived += mktH,
mktH => publisher.FeedArrived -= mktH
).Where((feed) => feed.SymbolId == symbolId)
select update.LTP;
//aggregates and yields results.
subs = eventAsObservable
.Scan <double, double>(0,
(acc, currentValue) =>
{
return(agg.Aggregate(currentValue));
}).Subscribe((state) => act(state, Constants.REDIS_MVA_ROOM_PREFIX + symbolId));
}
public static void AddAggregator(IDataPublisher publisher,
IAggregator<double, double> agg,
Action<double, string> act, int symbolId)
{
eventAsObservable = from update in Observable.FromEvent<OnFeedReceived, Feed>(
mktH => publisher.FeedArrived += mktH,
mktH => publisher.FeedArrived -= mktH
).Where((feed) => feed.SymbolId == symbolId)
select update.LTP;
//aggregates and yields results.
subs = eventAsObservable
.Scan<double, double>(0,
(acc, currentValue) =>
{
return agg.Aggregate(currentValue);
}).Subscribe((state) => act(state, Constants.REDIS_MVA_ROOM_PREFIX + symbolId));
}
public IList <Record> Search(Query query)
{
IList <IList <Record> > results = new List <IList <Record> >();
var tasks = new List <Task <IList <Record> > >();
Parallel.ForEach(
_scrapers,
scraper =>
{
try
{
results.Add(scraper.Search(query, PAGES));
}
catch (Exception)
{
}
});
return(_aggregator.Aggregate(results));
}
public async Task <List <T> > DoHttpRequestForSeveralRecipientsAsync <T>(HttpRequestInstructions <T> request, List <Uri> hosts, IAggregator <T> gatherer)
{
var listResults = new List <ResultEnvelope <T> >();
var listTasksToAwait = new List <Task>();
var cts = new CancellationTokenSource();
cts.CancelAfter(request.Timeout);
var testOutput = new StringBuilder();
foreach (var host in hosts)
{
try
{
var client = clientManager.GetClient(new HttpHost(host.OriginalString));
listTasksToAwait.Add(client.SendAsync(request.Request.Copy(), cts.Token).ContinueWith(
x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
testOutput.AppendLine($"this host worked fine: {host.AbsoluteUri}");
}
catch (Exception e)
{
testOutput.AppendLine("foreach Catch->" + e.Message);
}
}
try
{
Task.WaitAll(listTasksToAwait.ToArray());
//await Task.WhenAll(listTasksToAwait).ContinueWith(x => testOutput.AppendLine(x.Status.ToString()), TaskContinuationOptions.OnlyOnFaulted);
}
catch (Exception e) { testOutput.AppendLine("outer catch->" + e.Message); }
testOutput.AppendLine($"listResults.Count = {listResults.Count}; {string.Join(",", listResults.Select(x => x.Result))}");
listResults = listResults.Where(x => x != null).ToList();
testOutput.AppendLine($"listResults.Count = {listResults.Count}; {string.Join(",", listResults.Select(x => x.Result))}");
System.IO.File.WriteAllText(@"C:\users\az185030\desktop\sampletestresults.txt", testOutput.ToString());
var result = gatherer.Aggregate(listResults);
return(result.Select(x => x.Result).ToList());
}
public async Task <List <T> > DoSeveralHttpRequestsAsync <T>(List <HttpRequestInstructions <T> > requests, TimeSpan gatherTimeout, IAggregator <T> gatherer)
{
//TODO null checks all around
var listResults = new List <ResultEnvelope <T> >();
var listTasksToAwait = new List <Task>();
var cts = new CancellationTokenSource();
cts.CancelAfter(gatherTimeout);
foreach (var request in requests)
{
var client = clientManager.GetClient(new HttpHost(request.Request.RequestUri.ToString()));
if (client != null)
{
listTasksToAwait.Add(client.SendAsync(request.Request.Copy(), cts.Token).ContinueWith(
x => listResults.Add(request.ResponseTransformer(x.Result)), cts.Token));
}
}
await Task.WhenAll(listTasksToAwait).ConfigureAwait(false);
var result = gatherer.Aggregate(listResults);
return(result.Select(x => x.Result).ToList());
}
private static void AggregateAllDimensionsFlat(object aggregationContext, DataBlock block, ushort itemCount, ShortSet baseWhereSet, List <List <Tuple <IExpression, ShortSet> > > allDimensionValueSets, IUntypedColumn[] columns, IAggregator aggregator)
{
int nextBlockRow = 0;
int dimensionCount = allDimensionValueSets.Count;
int currentDimension = 0;
int[] nextIndexPerDimension = new int[dimensionCount];
ShortSet[] setsPerDimension = new ShortSet[dimensionCount + 1];
setsPerDimension[0] = baseWhereSet;
for (int i = 1; i < setsPerDimension.Length; ++i)
{
setsPerDimension[i] = new ShortSet(itemCount);
}
object[] aggregationResults = new object[block.RowCount];
while (currentDimension >= 0)
{
if (currentDimension == dimensionCount)
{
// Leaf: Compute Aggregate, then ask for next combination
aggregationResults[nextBlockRow] = aggregator.Aggregate(aggregationContext, setsPerDimension[currentDimension], columns);
nextBlockRow++;
currentDimension--;
}
else if (nextIndexPerDimension[currentDimension] < allDimensionValueSets[currentDimension].Count)
{
// Non-leaf, more values: Get next value and combine
Tuple <IExpression, ShortSet> dimensionValue = allDimensionValueSets[currentDimension][nextIndexPerDimension[currentDimension]];
ShortSet current = setsPerDimension[currentDimension + 1];
current.FromAnd(setsPerDimension[currentDimension], dimensionValue.Item2);
nextIndexPerDimension[currentDimension]++;
currentDimension++;
// Skip rest of range if set already empty [slower; IsEmpty too expensive]
if (current.IsEmpty())
{
int rowsToSkip = 1;
for (int i = currentDimension; i < dimensionCount; ++i)
{
rowsToSkip *= allDimensionValueSets[i].Count;
}
nextBlockRow += rowsToSkip;
currentDimension--;
}
}
else
{
// Non-leaf, no more values: Pop up to previous dimension for next value there
nextIndexPerDimension[currentDimension] = 0;
currentDimension--;
}
}
block.SetColumn(block.ColumnCount - 1, aggregationResults);
}
