private static void ActionBlockSample()
{
var processInput = new ActionBlock<string>(s =>
{
Console.WriteLine("user input: {0}", s);
});
processInput.Post("111");
processInput.Post("222");
}
public void TestPost()
{
foreach (bool bounded in DataflowTestHelpers.BooleanValues)
{
ActionBlock<int> ab = new ActionBlock<int>(i => { },
new ExecutionDataflowBlockOptions { BoundedCapacity = bounded ? 1 : -1 }); // test greedy and then non-greedy
Assert.True(ab.Post(0), "Expected non-completed ActionBlock to accept Post'd message");
ab.Complete();
Assert.False(ab.Post(0), "Expected Complete'd ActionBlock to decline messages");
}
}
public void Run()
{
this.output.WriteLine("TPL start");
this.CheckFileExists();
var f = new FileInfo(this.sampleLogFileName);
this.fileSizeInBytes = f.Length;
this.lineSizeInBytesSoFar = 0;
this.lineCount = 0;
var sw = new Stopwatch();
sw.Start();
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var ab = new ActionBlock<string>(s => this.ProcessLine(s), options);
using (TextReader reader = File.OpenText(this.sampleLogFileName))
{
string line;
while ((line = reader.ReadLine()) != null)
{
ab.Post(line);
}
}
ab.Complete();
ab.Completion.Wait();
sw.Stop();
this.ShowResults();
this.output.WriteLine();
this.output.WriteLine("TPL done in {0}", sw.Elapsed);
}
public void WriteOneLatency_OneMetricFlushed_Success()
{
_block = TimedLatencyAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
true,
_log.Object);
var pulseDate = DateTime.Now;
_block.Post(new Timing("foo.bar.baz", 100));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse(pulseDate);
Assert.AreEqual(new GraphiteLine("foo.bar.baz.count", 1, pulseDate.ToEpoch()),_outputBuffer.GetGraphiteLine(0));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.min", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(1));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.max", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(2));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.mean", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(3));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.sum", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(4));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.sumSquares", 10000, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(5));
Assert.AreEqual(6, _outputBuffer.GraphiteLines.Count);
// Ensure that min, max, mean and sum are all equal
Assert.IsTrue(_outputBuffer.GetGraphiteLine(1).Quantity == _outputBuffer.GetGraphiteLine(2).Quantity
&& _outputBuffer.GetGraphiteLine(2).Quantity == _outputBuffer.GetGraphiteLine(3).Quantity
&& _outputBuffer.GetGraphiteLine(3).Quantity == _outputBuffer.GetGraphiteLine(4).Quantity);
}
static void Test()
{
ConcurrentExclusiveSchedulerPair pair = new ConcurrentExclusiveSchedulerPair();
readerAB1 = new ActionBlock<int>((i) =>
{
Console.WriteLine(i + " ReaderAB1 begin handing." + " Excute Time:" + DateTime.Now +
" currentThreadId : " + Thread.CurrentThread.ManagedThreadId);
Thread.Sleep(500);
}, new ExecutionDataflowBlockOptions() {TaskScheduler = pair.ConcurrentScheduler});
readerAB2 = new ActionBlock<int>((i) =>
{
Console.WriteLine(i + " ReaderAB2 begin handing." + " Excute Time:" + DateTime.Now +
" currentThreadId : " + Thread.CurrentThread.ManagedThreadId);
Thread.Sleep(500);
}, new ExecutionDataflowBlockOptions() {TaskScheduler = pair.ConcurrentScheduler});
readerAB3 = new ActionBlock<int>((i) =>
{
Console.WriteLine(i + " ReaderAB3 begin handing." + " Excute Time:" + DateTime.Now +
" currentThreadId : " + Thread.CurrentThread.ManagedThreadId);
Thread.Sleep(500);
}, new ExecutionDataflowBlockOptions() {TaskScheduler = pair.ConcurrentScheduler});
writeAB1 = new ActionBlock<int>((i) =>
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(i + " writeAB1 begin handing." + " Excute Time:" + DateTime.Now +
" currentThreadId : " + Thread.CurrentThread.ManagedThreadId);
Console.ResetColor();
Thread.Sleep(3000);
}, new ExecutionDataflowBlockOptions() {TaskScheduler = pair.ExclusiveScheduler});
bb.LinkTo(readerAB1);
bb.LinkTo(readerAB2);
bb.LinkTo(readerAB3);
var task1 = Task.Run(() =>
{
while (true)
{
bb.Post(1);
Thread.Sleep(1000);
}
});
var task2 = Task.Run(() =>
{
while (true)
{
writeAB1.Post(1);
Thread.Sleep(6000);
}
});
Task.WaitAll(task1, task2);
}
public async Task TestBlockBufferCount()
{
var block1 = new TransformBlock<int, int>(i => 2 * i);
var block2 = new TransformManyBlock<int, int>(i => new [] { i });
var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); });
block1.Post(0);
block2.Post(0);
block2.Post(0);
block3.Post(0);
block3.Post(0);
block3.Post(0);
await Task.Delay(200);
Assert.AreEqual(1, block1.GetBufferCount().Total());
Assert.AreEqual(2, block2.GetBufferCount().Total());
Assert.AreEqual(2, block3.GetBufferCount().Total());
}
public void WriteFourLatencies_PercentileLogged_Success()
{
_block = TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
90,
null,
_log.Object);
_block.Post(new Timing("foo", 100));
_block.Post(new Timing("foo", 200));
_block.Post(new Timing("foo", 300));
_block.Post(new Timing("foo", 400));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse();
Assert.IsTrue(_outputBuffer.GraphiteLines.Any(p => p.Name == "foo.p90"));
Assert.AreEqual(400, _outputBuffer["foo.p90"]);
}
public void Start(string root)
{
_token = new CancellationTokenSource();
_task = Loop((now, ct) => SendMessagesAsync(ct), _token.Token);
_task.Post(DateTimeOffset.Now);
_root = root;
//_stopwatch.Start();
Frontier.Push(_root, null);
}
public void WriteTwoLatencies_CalulateMinMaxMeanSum_Success()
{
_block = TimedLatencyAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
true,
_log.Object);
var pulseDate = DateTime.Now;
_block.Post(new Timing("foo", 5));
_block.Post(new Timing("foo", 15));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse(pulseDate);
Assert.AreEqual(15, _outputBuffer["foo.max"]);
Assert.AreEqual(5, _outputBuffer["foo.min"]);
Assert.AreEqual(10, _outputBuffer["foo.mean"]);
Assert.AreEqual(2, _outputBuffer["foo.count"]);
Assert.AreEqual(20, _outputBuffer["foo.sum"]);
}
public async Task CachePage(EventPage page)
{
Accumulator.Store(page);
if (Accumulator.CachedEventCount > _projection.AsyncOptions.MaximumStagedEventCount)
{
_logger.ProjectionBackedUp(this, Accumulator.CachedEventCount, page);
await _fetcher.Pause().ConfigureAwait(false);
}
_executionTrack?.Post(page);
}
public void CompleteTest ()
{
var block = new ActionBlock<int> (i => Thread.Sleep (100));
for (int i = 0; i < 10; i++)
Assert.IsTrue (block.Post (i), "Not Accepted");
block.Complete ();
// Still element to be processed so Completion should be false
Assert.IsFalse (block.Completion.IsCompleted);
block.Completion.Wait ();
Assert.IsTrue (block.Completion.IsCompleted);
}
public void BasicUsageTest ()
{
bool[] array = new bool[3];
var evt = new CountdownEvent (array.Length);
var block = new ActionBlock<int> (i => { array[i] = true; evt.Signal (); });
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
Assert.IsTrue (evt.Wait (500));
Assert.IsTrue (array.All (b => b), "Some false");
}
public void WriteOneHunderedLatencies_p90PercentileLogged_Success()
{
_block = TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
90,
null,
_log.Object);
TestUtility.Range(100, false).ForEach(p => _block.Post(new Timing("foo", p)));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse();
Assert.AreEqual(90, _outputBuffer["foo.p90"]);
}
internal ProjectionUpdateBatch(EventGraph events, DocumentSessionBase session, EventRange range, CancellationToken token)
{
Range = range;
_session = session;
_token = token;
Queue = new ActionBlock <IStorageOperation>(processOperation,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, EnsureOrdered = true, CancellationToken = token
});
startNewPage(session);
var progressOperation = range.BuildProgressionOperation(events);
Queue.Post(progressOperation);
}
private static Task NonSequentialWithTdfAsync(int n)
{
var options = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = n
};
var abSequential = new ActionBlock <int>(SingleStepAsync,
options);
for (int i = 0; i < 10; i++)
{
abSequential.Post(i);
}
abSequential.Complete();
return(abSequential.Completion);
}
private static void SimpleActionBlock()
{
Console.WriteLine("SimpleActionBlock");
var ab = new ActionBlock <int>(x =>
{
Console.WriteLine($"({x}) Task Id: {Task.CurrentId}");
Thread.Sleep(10);
});
for (int i = 0; i < 10; i++)
{
ab.Post(i);
}
ab.Complete();
ab.Completion.Wait();
}
public void Configuration(IAppBuilder builder)
{
var config = new HttpConfiguration();
// Configure routing
config.MapHttpAttributeRoutes();
// Configure serialization
config.Formatters.XmlFormatter.UseXmlSerializer = true;
config.Formatters.JsonFormatter.SerializerSettings.ContractResolver
= new Newtonsoft.Json.Serialization.CamelCasePropertyNamesContractResolver();
config.Routes.MapHttpRoute("tradingApi", "api/trading/{id}",
new { controller = "Trading", id = RouteParameter.Optional });
config.Routes.MapHttpRoute("DefaultApi", "api/{controller}/{id}",
new { controller = "{controller}", id = RouteParameter.Optional });
// replace the default controller activator
config.Services.Replace(
typeof(IHttpControllerActivator),
new ControlActivatorPublisher(
Observer.Create <CommandWrapper>(
x => agent.Post <CommandWrapper>(x))));
// Enable Swagger and Swagger UI
config
.EnableSwagger(c => c.SingleApiVersion("v1", "StockTicker API"))
.EnableSwaggerUi();
var configSignalR = new HubConfiguration()
{
EnableDetailedErrors = true
}; // #E
GlobalHost.HubPipeline.AddModule(new ErrorHandlingPipelineModule());
Owin.CorsExtensions.UseCors(builder, Microsoft.Owin.Cors.CorsOptions.AllowAll);
TradingCoordinator.Instance()
.AddPublisher(StockMarket.StockMarket.Instance().AsObservable());
// MapSignalR() is an extension method of IAppBuilder provided by SignalR to facilitate mapping and configuration of the hub service.
// The generic overload MapSignalR<TConnection> is used to map persistent connections, that we do not have to specify the classes that implement the services
builder.MapSignalR(configSignalR);
builder.UseWebApi(config);
}
static async Task Main(string[] args)
{
var processedDelaysCollection = new BlockingCollection <int>();
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4
};
var actionBlock = new ActionBlock <int>(async delay =>
{
if (delay == 3 || delay == 5)
{
throw new ArgumentException($"Nope delay {delay} is not supported, not at all");
}
await Task.Delay(1000 * delay);
processedDelaysCollection.Add(delay);
}, options);
var stopWatch = Stopwatch.StartNew();
foreach (var delay in new int[] { 8, 6, 5, 3 })
{
actionBlock.Post(delay);
}
actionBlock.Complete();
try
{
await actionBlock.Completion;
}
catch (AggregateException errors)
{
foreach (var ex in errors.InnerExceptions)
{
Console.WriteLine($"{ex.Message}");
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex.Message}");
}
stopWatch.Stop();
Console.WriteLine($"Processing took {stopWatch.Elapsed} secs");
Console.WriteLine($"Processed delays : {string.Join(',', processedDelaysCollection)}");
}
private async void Execute()
{
var cancelToken = _cancellationTokenSource.Token;
var opts = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = MaxConcurrentItems,
BoundedCapacity = MaxConcurrentItems + 1
};
var actionBlock = new ActionBlock <string>(ExecuteItem, opts);
while (!cancelToken.IsCancellationRequested)
{
try
{
if (!SpinWait.SpinUntil(() => actionBlock.InputCount == 0, Options.IdleTime))
{
continue;
}
var item = await QueueProvider.DequeueWork(Queue, cancelToken);
if (item == null)
{
if (!QueueProvider.IsDequeueBlocking)
{
await Task.Delay(Options.IdleTime, cancelToken);
}
continue;
}
if (!actionBlock.Post(item))
{
await QueueProvider.QueueWork(item, Queue);
}
}
catch (OperationCanceledException)
{
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
}
}
actionBlock.Complete();
await actionBlock.Completion;
}
// other options for perf.
// 1) only grab certain directories either with dockerfiles or as specified by build.yaml
// 2) Prioritize large files or files with lots of copies.
// 3) parallelize copy with buffer first attempt at that with _contentClient.GetBufferAsync failed. Also lots of memory.
// 4) multistream copyasync
public async Task <Dictionary <string, double> > Materialize(string localDestination)
{
var uniqueblobs = _files.GroupBy(keySelector: file => file.Blob.Id, resultSelector: (key, file) => file).ToList();
var metrics = new Dictionary <string, double>
{
["files"] = _files.Count,
["uniqueblobs"] = uniqueblobs.Count
};
if (_computeDockerHashes)
{
ComputeDockerHashes(localDestination, metrics);
}
var dltimes = new ConcurrentBag <double>();
var copytimes = new ConcurrentBag <double>();
var filesize = new ConcurrentBag <double>();
var throttler = new ActionBlock <IEnumerable <VstsFile> >(list => DownloadGrouping(list, localDestination, dltimes, copytimes, filesize), new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _concurrentDownloads
});
foreach (var grouping in uniqueblobs)
{
throttler.Post(grouping);
}
throttler.Complete();
await throttler.Completion;
if (dltimes.Any())
{
metrics["DownloadCount"] = dltimes.Count();
metrics["AverageDownloadSecs"] = dltimes.Average();
metrics["MaxDownloadSecs"] = dltimes.Max();
CalculatePercentile(metrics, dltimes, "Download");
}
if (copytimes.Any())
{
metrics["CopyCount"] = copytimes.Count();
metrics["AverageCopySecs"] = copytimes.Average();
metrics["MaxCopySecs"] = copytimes.Max();
CalculatePercentile(metrics, copytimes, "CopyTime");
}
return(metrics);
}
/// <summary>
/// Acknowledge a response from a fetch request. Will post a corresponding
/// message to the inner actor.
/// </summary>
/// <param name="fetchResponse">a fetch response from a Kafka broker.</param>
public void Acknowledge(CommonAcknowledgement <FetchPartitionResponse> fetchResponse)
{
_innerActor.Post(new ConsumerMessage
{
MessageType = ConsumerMessageType.FetchResponse,
MessageValue = new ConsumerMessageValue
{
FetchResponse = fetchResponse
}
});
}
/// <summary>
/// Starts a periodic task with delay that gets cancelled using cancellation tokens.
/// To start the task: PeriodictaskWithDelay(action, token, delay).Post(DateTimeOffset.Now);
/// </summary>
/// <param name="action">The work to perform.</param>
/// <param name="cancellationToken">The cancellation token to stop the loop.</param>
/// <param name="delay">The delay between each execution (milliseconds)</param>
/// <returns>The target action block.</returns>
public static ActionBlock <DateTimeOffset> PeriodicTaskWithDelay(
Action <DateTimeOffset> action, CancellationToken cancellationToken, int delay)
{
ActionBlock <DateTimeOffset> block = null;
block = new ActionBlock <DateTimeOffset>(async now => {
action(now);
await Task.Delay(TimeSpan.FromMilliseconds(delay), cancellationToken).
ConfigureAwait(false);
block.Post(DateTimeOffset.Now);
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken
});
return(block);
}
static void Main(string[] args)
{
var actionBlock = new ActionBlock <int>(n =>
{
Task.Delay(500).Wait();
Console.WriteLine(n);
});
for (int i = 0; i < 10; i++)
{
actionBlock.Post(i);
Console.WriteLine($"There are {actionBlock.InputCount} in the input queue");
}
Console.WriteLine("done");
Console.ReadKey();
}
static void Main()
{
var ab = new ActionBlock <int>(i =>
{
Console.Write($"{i}) Task Id:{Task.CurrentId}\t");
Thread.Sleep(10);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4
});
for (int i = 1; i <= 10; i++)
{
ab.Post(i);
}
ab.Complete();
ab.Completion.Wait();
}
public void AsyncNullTest()
{
var scheduler = new TestScheduler ();
var block = new ActionBlock<int> (
i => null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
public async Task DoWorkLoads(List <IPAddress> addresses)
{
var options = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 255
};
var block = new ActionBlock <IPAddress>(TestSocket, options);
foreach (var ip in addresses)
{
block.Post(ip);
}
block.Complete();
await block.Completion;
}
public static ActionBlock <DateTimeOffset> Create(Func <DateTimeOffset, CancellationToken, Task> action, CancellationToken cancellationToken, TimeSpan delay)
{
// Validate parameters.
if (action == null)
{
throw new ArgumentNullException(nameof(action));
}
// Declare the block variable, it needs to be captured.
ActionBlock <DateTimeOffset> block = null;
// Create the block, it will call itself, so
// you need to separate the declaration and
// the assignment.
// Async so you can wait easily when the
// delay comes.
block = new ActionBlock <DateTimeOffset>(async now =>
{
// Perform the action. Wait on the result.
await action(now, cancellationToken).
// Doing this here because synchronization context more than
// likely *doesn't* need to be captured for the continuation
// here. As a matter of fact, that would be downright
// dangerous.
ConfigureAwait(false);
// Wait.
try
{
await Task.Delay(delay, cancellationToken)
.
// Same as above.
ConfigureAwait(false);
}
catch (OperationCanceledException) {}
// Post the action back to the block.
block.Post(DateTimeOffset.Now);
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken
});
// Return the block.
return(block);
}
public override void Run()
{
// 指定块最大并行数量为 4
ActionBlock <RunModel> actionBlock = new ActionBlock <RunModel>(
(model) => Helper.PrintLine($"ActionBlock: {model.Name}"),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4
});
var models = this.CreateCollection();
Parallel.ForEach(models, model => actionBlock.Post(model));
// 将块置为完成，并等待完成
actionBlock.Complete();
actionBlock.Completion.Wait();
}
/// <summary>
/// Executes given lambda parallely on given data set with max degree of parallelism set up
/// </summary>
/// <typeparam name="T">The item type</typeparam>
/// <param name="source">Data to process</param>
/// <param name="body">Lambda to execute on all items</param>
/// <param name="maxDegreeOfParallelism">Max degree of parallelism (-1 for unbounded execution)</param>
/// <returns></returns>
internal static Task AsyncParallelForEach <T>(this IEnumerable <T> source, Func <T, Task> body, int maxDegreeOfParallelism)
{
var options = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
};
var block = new ActionBlock <T>(body, options);
foreach (var item in source)
{
block.Post(item);
}
block.Complete();
return(block.Completion);
}
private static Task BatchAsync(CancellationToken cancellationToken)
{
var options = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 3,
CancellationToken = cancellationToken
};
var ab = new ActionBlock <int>(i => DownloadAndSave(i, cancellationToken)
, options);
for (int i = 0; i < 100; i++)
{
ab.Post(i);
}
ab.Complete();
return(ab.Completion);
}
public void ProcessAsync(Func <Task> func, LockType lockType)
{
switch (lockType)
{
case LockType.None:
func();
break;
case LockType.Read:
_readLockedQueue.Post(func);
break;
case LockType.Write:
_writeLockedQueue.Post(func);
break;
}
}
private void Flush(Dictionary <String, IDocumentDatabase> from, Dictionary <String, IDocumentDatabase> to)
{
var flush = new ActionBlock <FlushTo>(s =>
{
try
{
s.To.DropAndPrepare();
if (s.Data.Count > 0)
{
s.To.Save(s.Data);
}
}
catch (Exception ex)
{
_logger.Fatal(ex);
}
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 10
});
foreach (var inMemoryDatabasePair in from)
{
var inMemoryDatabase = (InMemoryDatabase)inMemoryDatabasePair.Value;
var destinationDatabase = to[inMemoryDatabasePair.Key];
destinationDatabase.DropAllCollections();
foreach (var inMemoryCollectionPair in inMemoryDatabase.Collections)
{
var inMemoryCollection = (IInMemoryCollection)inMemoryCollectionPair.Value;
var documentType = inMemoryCollectionPair.Key.DocumentType;
var collectionName = inMemoryCollectionPair.Key.CollectionName;
var destinationCollection = destinationDatabase.GetCollection(documentType, collectionName);
destinationCollection.DropAndPrepare();
flush.Post(new FlushTo(destinationCollection, inMemoryCollection.Documents.Values));
}
}
flush.Complete();
flush.Completion.Wait();
}
public async Task <long> Start(ProjectionDaemon daemon)
{
_logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name);
var singleFileOptions = new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
};
_commandBlock = new ActionBlock <Command>(processCommand, singleFileOptions);
_loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFileOptions);
_tracker = daemon.Tracker;
_daemon = daemon;
_fetcher = new EventFetcher(_store, _projectionShard.EventFilters);
_grouping = new TransformBlock <EventRange, EventRangeGroup>(groupEventRange, singleFileOptions);
_building = new ActionBlock <EventRangeGroup>(processRange, singleFileOptions);
_grouping.LinkTo(_building);
_loader.LinkTo(_grouping, e => e.Events.Any());
var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation);
foreach (var storageType in _source.Options.StorageTypes)
{
// TODO -- this will have to get fancier when we do multi-tenancy by database
_store.Tenancy.Default.EnsureStorageExists(storageType);
}
_commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted));
_subscription = _tracker.Subscribe(this);
_logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark);
Status = AgentStatus.Running;
Position = lastCommitted;
return(lastCommitted);
}
private static void Main()
{
var actionBlock = new ActionBlock <int>(i =>
{
Thread.Sleep(1000);
Console.WriteLine(i);
});
for (var i = 0; i < 10; i++)
{
actionBlock.Post(i);
}
Console.WriteLine("Done");
Console.ReadKey();
}
ITargetBlock <IEnumerable <ArbitragePair> > CreateNeverEndingTask(
Func <IEnumerable <ArbitragePair>, CancellationToken, Task> action,
CancellationToken cancellationToken)
{
// Validate parameters.
if (action == null)
{
throw new ArgumentNullException("action");
}
// Declare the block variable, it needs to be captured.
ActionBlock <IEnumerable <ArbitragePair> > block = null;
// Create the block, it will call itself, so
// you need to separate the declaration and
// the assignment.
// Async so you can wait easily when the
// delay comes.
block = new ActionBlock <IEnumerable <ArbitragePair> >(async pairs =>
{
// Perform the action. Wait on the result.
await action(pairs, cancellationToken).
// Doing this here because synchronization context more than
// likely *doesn't* need to be captured for the continuation
// here. As a matter of fact, that would be downright
// dangerous.
ConfigureAwait(false);
// Wait.
await Task.Delay(TimeSpan.FromSeconds(300), cancellationToken).
// Same as above.
ConfigureAwait(false);
pairs = dbService.GetArbitragePairs("", baseExchange.Name, counterExchange.Name).Select(p => new ArbitragePair(p));
// Post the action back to the block.
block.Post(pairs);
//block.Complete();
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken
});
// Return the block.
return(block);
}
/// <summary>
/// Lints a given set of specs that belong to a workspace and include any potential linter failures in the resulting one.
/// </summary>
public static Workspace CreateLintedWorkspaceForChangedSpecs(
Workspace workspace,
IEnumerable <ISourceFile> changedSpecsToLint,
LoggingContext loggingContext,
IFrontEndConfiguration configuration,
PathTable pathTable)
{
var logger = BuildXL.FrontEnd.Script.Tracing.Logger.CreateLogger(preserveLogEvents: true);
var linter = DiagnosticAnalyzer.Create(
logger,
loggingContext,
new HashSet <string>(configuration.EnabledPolicyRules),
disableLanguagePolicies: false);
// Lint all files in parallel and wait for queue completion
var linterQueue = new ActionBlock <ISourceFile>(
(Action <ISourceFile>)LintFile,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = configuration.MaxFrontEndConcurrency()
});
foreach (var sourceFile in changedSpecsToLint)
{
linterQueue.Post(sourceFile);
}
linterQueue.Complete();
linterQueue.Completion.GetAwaiter().GetResult();
// Create a workspace with the extra failures (if any) and return it
var linterFailures = ComputeLinterFailures(workspace, logger, pathTable);
return(linterFailures.Count > 0 ? workspace.WithExtraFailures(linterFailures) : workspace);
// Local functions
void LintFile(ISourceFile sourceFile)
{
if (!sourceFile.HasDiagnostics())
{
linter.AnalyzeFile(sourceFile, logger, loggingContext, pathTable, workspace);
}
}
}
public static async Task Main(string[] args)
{
var service = new PerformanceService(null);
if (args.Length < 1)
{
Console.WriteLine("url is required");
return;
}
var url = args[0];
var size = 100000;
if (args.Length > 1)
{
size = int.Parse(args[1]);
}
var degree = 10;
if (args.Length > 2)
{
degree = int.Parse(args[2]);
}
var handler = new HttpPerformanceDelegatingHandler(new HttpClientHandler());
var client = new HttpClient(handler);
var block = new ActionBlock <int>(async i => await client.GetAsync(string.Format(url, i)), new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = degree
});
foreach (var i in Enumerable.Range(0, size))
{
block.Post(i);
}
block.Complete();
await block.Completion;
}
private static void DataStructure_ActionBlock()
{
// Create an ActionBlock<int> object that prints values
// to the console.
var actionBlock = new ActionBlock <int>(n =>
Console.WriteLine(n));
// Post several messages to the block.
for (int i = 0; i < 3; i++)
{
actionBlock.Post(i * 10);
}
// Set the block to the completed state and wait for all
// tasks to finish.
actionBlock.Complete();
actionBlock.Completion.Wait();
}
/// <inheritdoc />
public override async Task SendWebHookWorkItemsAsync(IEnumerable <WebHookWorkItem> workItems)
{
if (workItems == null)
{
throw new ArgumentNullException(nameof(workItems));
}
_launcher = new ActionBlock <WebHookWorkItem>(async item => {
await LaunchWebHook(item);
}
, _options);
foreach (WebHookWorkItem workItem in workItems)
{
_launcher.Post(workItem);
}
_launcher.Complete();
await _launcher.Completion;
}
private static void Main(string[] args)
{
//UnhandledException();
// InternalExceptionHandling();
//ExternallyExceptionHandling();
PropagatingAndHandlingExceptions();
var divideBlock =
new ActionBlock <Tuple <int, int> >(
(Action <Tuple <int, int> >)(input =>
{
Thread.Sleep(10);
Console.WriteLine(input.Item1 / input.Item2);
}),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 2
});
divideBlock.Post(Tuple.Create(10, 5));
divideBlock.Post(Tuple.Create(20, 4));
divideBlock.Post(Tuple.Create(10, 0));
divideBlock.Post(Tuple.Create(10, 2));
divideBlock.Post(Tuple.Create(50, 5));
divideBlock.Post(Tuple.Create(10, 5));
divideBlock.Post(Tuple.Create(20, 4));
// Console.Read();
try
{
divideBlock.Completion.Wait();
}
catch (AggregateException errors)
{
foreach (Exception error in errors.InnerExceptions)
{
Console.WriteLine(error.Message);
}
}
}
/// <summary>
/// This constructor should be used for asynchronous resource creation
/// </summary>
/// <param name="resourceTaskFactory">Function to create a task that returns a resource</param>
/// <param name="options"></param>
public AsyncResourcePool(Func <Task <TResource> > resourceTaskFactory, AsyncResourcePoolOptions options)
{
_minNumResources = options.MinNumResources;
_maxNumResources = options.MaxNumResources;
_resourcesExpireAfter = options.ResourcesExpireAfter;
_maxNumResourceCreationAttempts = options.MaxNumResourceCreationAttempts;
_resourceCreationRetryInterval = options.ResourceCreationRetryInterval;
_availableResources = new Queue <TimestampedResource>();
_pendingResourceRequests = new Queue <ResourceRequestMessage>();
_resourceTaskFactory = resourceTaskFactory;
// Important: These functions must be called after all instance members have been initialised!
_messageHandler = GetMessageHandler();
SetupErrorHandling();
SetupPeriodicPurge();
_messageHandler.Post(new EnsureAvailableResourcesMessage());
}
/// <summary>
/// Process the data.
/// </summary>
public static void ProcessData <T>
(
[NotNull] this IEnumerable <T> data,
[NotNull] ActionBlock <T> actionBlock
)
{
Code.NotNull(data, "data");
Code.NotNull(actionBlock, "actionBlock");
foreach (T item in data)
{
actionBlock.Post(item);
//actionBlock.SendAsync(item);
}
actionBlock.Complete();
actionBlock.Completion.Wait();
}
private static async Task TrdStageAsync()
{
// make the rest of the code async
// otherwise you will have dead lock (GetConsumingEnumerable is
// blocking API therefore could be dangerous)
await Task.Delay(1);
foreach (Completeble <string> message in _secondQueue.GetConsumingEnumerable())
{
await Task.Delay(2000);
string data = $"{message.Value} -> third stage";
var nextMessage = message.ProceedWith(data);
_finalProcesing.Post(nextMessage); // enqueue and complete
Console.Write(" 3rd ");
}
}
private static void ActionBlockSample()
{
var processInput = new ActionBlock<string>(s =>
{
Console.WriteLine("user input: {0}", s);
});
bool exit = false;
while (!exit)
{
string input = Console.ReadLine();
if (string.Compare(input, "exit", ignoreCase: true) == 0)
{
exit = true;
}
else
{
processInput.Post(input);
}
}
}
internal Task SendBatchAsync(IEnumerable<BrokeredMessage> messages)
{
var postBlock = new ActionBlock<IGrouping<string, BrokeredMessage>>((group) =>
{
var r = R.Next(_scaleCount);
TopicClient client = GetClient(group.Key, r);
Logger.TraceFormat("Posting Messages onto Topic {1} '{0}'", client.Path, r);
return client.SendBatchAsync(messages);
});
foreach (var group in messages.GroupBy(m => m.CorrelationId ?? DEFAULT_COORELATION_ID))
postBlock.Post(group);
postBlock.Complete();
return postBlock.Completion;
}
//[Fact(Skip = "Outerloop")]
public void RunActionBlockConformanceTests()
{
// SYNC
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> actionBlockFactory =
options =>
{
ITargetBlock<int> target = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
return new TargetProperties<int> { Target = target, Capturer = target, ErrorVerifyable = true };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 1 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 1, CancellationToken = cancellationSource.Token };
var spscOptions = new ExecutionDataflowBlockOptions { SingleProducerConstrained = true };
var spscMptOptions = new ExecutionDataflowBlockOptions { SingleProducerConstrained = true, MaxMessagesPerTask = 10 };
Assert.True(FeedTarget(actionBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, spscOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, spscOptions, 1, Intervention.Complete, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, spscMptOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, spscMptOptions, 1, Intervention.Complete, null, feedMethod, true));
}
// Test scheduler usage
{
bool localPassed = true;
for (int trial = 0; trial < 2; trial++)
{
var sts = new SimpleTaskScheduler();
var options = new ExecutionDataflowBlockOptions { TaskScheduler = sts, MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded, MaxMessagesPerTask = 1 };
if (trial == 0) options.SingleProducerConstrained = true;
var ab = new ActionBlock<int>(i => localPassed &= TaskScheduler.Current.Id == sts.Id, options);
for (int i = 0; i < 2; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
}
Assert.True(localPassed, string.Format("{0}: Correct scheduler usage", localPassed ? "Success" : "Failure"));
}
// Test count
{
bool localPassed = true;
for (int trial = 0; trial < 2; trial++)
{
var barrier1 = new Barrier(2);
var barrier2 = new Barrier(2);
var ab = new ActionBlock<int>(i =>
{
barrier1.SignalAndWait();
barrier2.SignalAndWait();
}, new ExecutionDataflowBlockOptions { SingleProducerConstrained = (trial == 0) });
for (int iter = 0; iter < 2; iter++)
{
for (int i = 1; i <= 2; i++) ab.Post(i);
for (int i = 1; i >= 0; i--)
{
barrier1.SignalAndWait();
localPassed &= i == ab.InputCount;
barrier2.SignalAndWait();
}
}
}
Assert.True(localPassed, string.Format("{0}: InputCount", localPassed ? "Success" : "Failure"));
}
// Test ordering
{
bool localPassed = true;
for (int trial = 0; trial < 2; trial++)
{
int prev = -1;
var ab = new ActionBlock<int>(i =>
{
if (prev + 1 != i) localPassed &= false;
prev = i;
}, new ExecutionDataflowBlockOptions { SingleProducerConstrained = (trial == 0) });
for (int i = 0; i < 2; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
}
Assert.True(localPassed, string.Format("{0}: Correct ordering", localPassed ? "Success" : "Failure"));
}
// Test non-greedy
{
bool localPassed = true;
var barrier = new Barrier(2);
var ab = new ActionBlock<int>(i =>
{
barrier.SignalAndWait();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
ab.SendAsync(1);
Task.Delay(200).Wait();
var sa2 = ab.SendAsync(2);
localPassed &= !sa2.IsCompleted;
barrier.SignalAndWait(); // for SendAsync(1)
barrier.SignalAndWait(); // for SendAsync(2)
localPassed &= sa2.Wait(100);
int total = 0;
ab = new ActionBlock<int>(i =>
{
Interlocked.Add(ref total, i);
Task.Delay(1).Wait();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
for (int i = 1; i <= 100; i++) ab.SendAsync(i);
SpinWait.SpinUntil(() => total == ((100 * 101) / 2), 30000);
localPassed &= total == ((100 * 101) / 2);
Assert.True(localPassed, string.Format("total={0} (must be {1})", total, (100 * 101) / 2));
Assert.True(localPassed, string.Format("{0}: Non-greedy support", localPassed ? "Success" : "Failure"));
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
for (int trial = 0; trial < 2; trial++)
{
int sumOfOdds = 0;
var ab = new ActionBlock<int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
sumOfOdds += i;
}, new ExecutionDataflowBlockOptions { SingleProducerConstrained = (trial == 0) });
for (int i = 0; i < 4; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
localPassed = sumOfOdds == (1 + 3);
}
Assert.True(localPassed, string.Format("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var ab = new ActionBlock<int>(i => { }, dbo);
localPassed &= ab.Post(42) == false;
localPassed &= ab.InputCount == 0;
localPassed &= ab.Completion != null;
ab.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
// Test faulting
{
bool localPassed = true;
for (int trial = 0; trial < 2; trial++)
{
var ab = new ActionBlock<int>(i => { throw new InvalidOperationException(); },
new ExecutionDataflowBlockOptions { SingleProducerConstrained = (trial == 0) });
ab.Post(42);
ab.Post(1);
ab.Post(2);
ab.Post(3);
try { localPassed &= ab.Completion.Wait(5000); }
catch { }
localPassed &= ab.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => ab.InputCount == 0, 500);
localPassed &= ab.Post(4) == false;
}
Assert.True(localPassed, string.Format("{0}: Faulted handled correctly", localPassed ? "Success" : "Failure"));
}
// ASYNC (a copy of the sync but with constructors returning Task instead of void
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> actionBlockFactory =
options =>
{
ITargetBlock<int> target = new ActionBlock<int>(i => TrackCapturesAsync(i), (ExecutionDataflowBlockOptions)options);
return new TargetProperties<int> { Target = target, Capturer = target, ErrorVerifyable = true };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
Assert.True(FeedTarget(actionBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true));
Assert.True(FeedTarget(actionBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true));
}
// Test scheduler usage
{
bool localPassed = true;
var sts = new SimpleTaskScheduler();
var ab = new ActionBlock<int>(i =>
{
localPassed &= TaskScheduler.Current.Id == sts.Id;
return Task.Factory.StartNew(() => { });
}, new ExecutionDataflowBlockOptions { TaskScheduler = sts, MaxDegreeOfParallelism = -1, MaxMessagesPerTask = 10 });
for (int i = 0; i < 2; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
Assert.True(localPassed, string.Format("{0}: Correct scheduler usage", localPassed ? "Success" : "Failure"));
}
// Test count
{
bool localPassed = true;
var barrier1 = new Barrier(2);
var barrier2 = new Barrier(2);
var ab = new ActionBlock<int>(i => Task.Factory.StartNew(() =>
{
barrier1.SignalAndWait();
barrier2.SignalAndWait();
}));
for (int iter = 0; iter < 2; iter++)
{
for (int i = 1; i <= 2; i++) ab.Post(i);
for (int i = 1; i >= 0; i--)
{
barrier1.SignalAndWait();
localPassed &= i == ab.InputCount;
barrier2.SignalAndWait();
}
}
Assert.True(localPassed, string.Format("{0}: InputCount", localPassed ? "Success" : "Failure"));
}
// Test ordering
{
bool localPassed = true;
int prev = -1;
var ab = new ActionBlock<int>(i =>
{
return Task.Factory.StartNew(() =>
{
if (prev + 1 != i) localPassed &= false;
prev = i;
});
});
for (int i = 0; i < 2; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
Assert.True(localPassed, string.Format("{0}: Correct ordering", localPassed ? "Success" : "Failure"));
}
// Test non-greedy
{
bool localPassed = true;
var barrier = new Barrier(2);
var ab = new ActionBlock<int>(i =>
{
return Task.Factory.StartNew(() =>
{
barrier.SignalAndWait();
});
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
ab.SendAsync(1);
Task.Delay(200).Wait();
var sa2 = ab.SendAsync(2);
localPassed &= !sa2.IsCompleted;
barrier.SignalAndWait(); // for SendAsync(1)
barrier.SignalAndWait(); // for SendAsync(2)
localPassed &= sa2.Wait(100);
int total = 0;
ab = new ActionBlock<int>(i =>
{
return Task.Factory.StartNew(() =>
{
Interlocked.Add(ref total, i);
Task.Delay(1).Wait();
});
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
for (int i = 1; i <= 100; i++) ab.SendAsync(i);
SpinWait.SpinUntil(() => total == ((100 * 101) / 2), 30000);
localPassed &= total == ((100 * 101) / 2);
Assert.True(localPassed, string.Format("total={0} (must be {1})", total, (100 * 101) / 2));
Assert.True(localPassed, string.Format("{0}: Non-greedy support", localPassed ? "Success" : "Failure"));
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
int sumOfOdds = 0;
var ab = new ActionBlock<int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return Task.Factory.StartNew(() => { sumOfOdds += i; });
});
for (int i = 0; i < 4; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
localPassed = sumOfOdds == (1 + 3);
Assert.True(localPassed, string.Format("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"));
}
// Test that null task is ignored
{
bool localPassed = true;
int sumOfOdds = 0;
var ab = new ActionBlock<int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => { sumOfOdds += i; });
});
for (int i = 0; i < 4; i++) ab.Post(i);
ab.Complete();
ab.Completion.Wait();
localPassed = sumOfOdds == (1 + 3);
Assert.True(localPassed, string.Format("{0}: null tasks are ignored", localPassed ? "Success" : "Failure"));
}
// Test faulting from the delegate
{
bool localPassed = true;
var ab = new ActionBlock<int>(new Func<int, Task>(i => { throw new InvalidOperationException(); }));
ab.Post(42);
ab.Post(1);
ab.Post(2);
ab.Post(3);
try { localPassed &= ab.Completion.Wait(100); }
catch { }
localPassed &= ab.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => ab.InputCount == 0, 500);
localPassed &= ab.Post(4) == false;
Assert.True(localPassed, string.Format("{0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"));
}
// Test faulting from the task
{
bool localPassed = true;
var ab = new ActionBlock<int>(i => Task.Factory.StartNew(() => { throw new InvalidOperationException(); }));
ab.Post(42);
ab.Post(1);
ab.Post(2);
ab.Post(3);
try { localPassed &= ab.Completion.Wait(100); }
catch { }
localPassed &= ab.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => ab.InputCount == 0, 500);
localPassed &= ab.Post(4) == false;
Assert.True(localPassed, string.Format("{0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"));
}
}
async Task TransformCoreAsync(StreamReader reader, StreamWriter writer)
{
var writerBlock = new ActionBlock<string>(l => writer.WriteLineAsync(l));
for (; ; )
{
var line = await reader.ReadLineAsync().ConfigureAwait(false);
if (null == line)
break;
line = line.Normalize(NormalizationForm.FormC);
line = line.TrimEnd();
writerBlock.Post(line);
}
writerBlock.Complete();
await writerBlock.Completion.ConfigureAwait(false);
}
public void DefaultSchedulerIsDefaultTest ()
{
var scheduler = new TestScheduler ();
var factory = new TaskFactory (scheduler);
ActionBlock<int> action = null;
var task = factory.StartNew (() =>
{
Assert.AreEqual (scheduler, TaskScheduler.Current);
action = new ActionBlock<int> (
i => Assert.AreNotEqual (scheduler, TaskScheduler.Current));
Assert.IsTrue (action.Post (1));
action.Complete ();
});
Assert.AreEqual (1, scheduler.ExecuteAll ());
Assert.IsNotNull (action);
Assert.IsTrue (action.Completion.Wait (1000));
Assert.IsTrue (task.Wait (0));
}
private static IEnumerable<string> ConvertToCsvLines(IEnumerable<CloudBlockBlob> blobs, int maxDegreeOfParallelism)
{
var lines = new BlockingCollection<string>(BoundedCapacity);
Task.Run(async () =>
{
var actionBlock = new ActionBlock<CloudBlockBlob>(
async (b) =>
{
var line = await ConvertToCsvLineAsync(b);
lines.Add(line);
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism });
foreach (var blob in blobs)
{
var postSuccess = actionBlock.Post(blob);
}
actionBlock.Complete();
await actionBlock.Completion;
lines.CompleteAdding();
});
return lines.GetConsumingEnumerable();
}
public void TestEncapsulate_LinkingAndUnlinking()
{
var buffer = new BufferBlock<int>();
var action = new ActionBlock<int>(i => buffer.Post(i));
action.Completion.ContinueWith(delegate { buffer.Complete(); }, TaskScheduler.Default);
IPropagatorBlock<int, int> encapsulated = DataflowBlock.Encapsulate(action, buffer);
var endTarget = new BufferBlock<int>();
IDisposable unlink = encapsulated.LinkTo(endTarget);
Assert.NotNull(unlink);
IDisposable unlink2 = encapsulated.LinkTo(endTarget);
Assert.NotNull(unlink);
action.Post(1);
Assert.Equal(expected: 1, actual: endTarget.Receive());
unlink.Dispose();
action.Post(2);
Assert.Equal(expected: 2, actual: endTarget.Receive());
unlink2.Dispose();
}
/// <summary>
/// 请求队列处理程序
/// </summary>
/// <param name="dr"></param>
/// <param name="parseQueue"></param>
public void RequestHandler(DataRequest dr, ActionBlock<DataResponse> parseQueue)
{
//TODO [待完善]异常处理
var dcp = _dcpList[dr.GUID];
var dresp = dcp.SendHandler(dr);
parseQueue.Post(dresp);
}
internal static bool ActionPingPong()
{
const int ITERS = 2;
using (ManualResetEventSlim mres = new ManualResetEventSlim())
{
ActionBlock<int> c1 = null, c2 = null;
c1 = new ActionBlock<int>(i => c2.Post(i + 1));
c2 = new ActionBlock<int>(i =>
{
if (i >= ITERS) mres.Set();
else c1.Post(i + 1);
});
c1.Post(0);
mres.Wait();
return true;
}
}
public void AsyncCancelledTest()
{
var scheduler = new TestScheduler ();
var block = new ActionBlock<int> (
i =>
{
var tcs = new TaskCompletionSource<int> ();
tcs.SetCanceled ();
return tcs.Task;
}, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
}
public async Task TestFaulting()
{
for (int trial = 0; trial < 3; trial++)
foreach (bool singleProducerConstrained in DataflowTestHelpers.BooleanValues)
{
var options = new ExecutionDataflowBlockOptions { SingleProducerConstrained = singleProducerConstrained };
Action thrower = () => { throw new InvalidOperationException(); };
ActionBlock<int> ab = null;
switch (trial)
{
case 0: ab = new ActionBlock<int>(i => thrower(), options); break;
case 1: ab = new ActionBlock<int>(i => { thrower(); return Task.FromResult(0); }, options); break;
case 2: ab = new ActionBlock<int>(i => Task.Run(thrower), options); break;
}
for (int i = 0; i < 4; i++)
{
ab.Post(i); // Post may return false, depending on race with ActionBlock faulting
}
try
{
await ab.Completion;
Assert.True(false, "Should always throw IOE");
}
catch (InvalidOperationException) { }
Assert.Equal(expected: 0, actual: ab.InputCount);
Assert.False(ab.Post(5));
}
}
public async Task TestExceptionDataStorage()
{
const string DataKey = "DataflowMessageValue"; // must match key used in dataflow source
// Validate that a message which causes the ActionBlock to fault
// ends up being stored (ToString) in the resulting exception's Data
var ab1 = new ActionBlock<int>((Action<int>)(i => { throw new FormatException(); }));
ab1.Post(42);
await Assert.ThrowsAsync<FormatException>(() => ab1.Completion);
AggregateException e = ab1.Completion.Exception;
Assert.Equal(expected: 1, actual: e.InnerExceptions.Count);
Assert.Equal(expected: "42", actual: (string)e.InnerException.Data[DataKey]);
// Test case where message's ToString throws
var ab2 = new ActionBlock<ObjectWithFaultyToString>((Action<ObjectWithFaultyToString>)(i => { throw new FormatException(); }));
ab2.Post(new ObjectWithFaultyToString());
Exception ex = await Assert.ThrowsAsync<FormatException>(() => ab2.Completion);
Assert.False(ex.Data.Contains(DataKey));
}
public async Task TestFaultyScheduler()
{
var ab = new ActionBlock<int>(i => { },
new ExecutionDataflowBlockOptions
{
TaskScheduler = new DelegateTaskScheduler
{
QueueTaskDelegate = delegate { throw new FormatException(); }
}
});
ab.Post(42);
await Assert.ThrowsAsync<TaskSchedulerException>(() => ab.Completion);
}
public async Task ActionPingPong()
{
var tcs = new TaskCompletionSource<bool>();
ActionBlock<int> c1 = null, c2 = null;
c1 = new ActionBlock<int>(i => c2.Post(i + 1));
c2 = new ActionBlock<int>(i => {
if (i >= Iterations) tcs.SetResult(true);
else c1.Post(i + 1);
});
c1.Post(0);
await tcs.Task;
}
public void WriteLatenciesToTwoBuckets_MeasurementsSeparate_Success()
{
_block = TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
80,
null,
_log.Object);
var pulseDate = DateTime.Now;
// Bucket one
TestUtility.Range(5, false).ForEach(p => _block.Post(new Timing("foo", p * 100)));
// Bucket two
TestUtility.Range(5, false).ForEach(p => _block.Post(new Timing("bar", p * 100)));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse(pulseDate);
Assert.AreEqual(400, _outputBuffer["foo.p80"]);
Assert.AreEqual(400, _outputBuffer["bar.p80"]);
Assert.AreEqual(1, _outputBuffer.Items.Count);
}
public void TaskSchedulerTest ()
{
var scheduler = new TestScheduler ();
int n = 0;
var action = new ActionBlock<int> (
i => Interlocked.Increment (ref n),
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (action.Post (1));
Assert.AreEqual (0, Volatile.Read (ref n));
Assert.AreEqual (1, scheduler.ExecuteAll ());
Assert.AreEqual (1, Volatile.Read (ref n));
}
/// <summary>
/// 解析队列处理程序
/// </summary>
/// <param name="dresp"></param>
/// <param name="storeQueue"></param>
public void ParseHandler(DataResponse dresp, ActionBlock<DataContainer> storeQueue)
{
//TODO [待完善]异常处理
var dcp = _dcpList[dresp.GUID];
DataContainer dc;
try
{
dc = dcp.ParseHandler(dresp);
}
catch (ParseErrorException)
{
//TODO [待完善]统一保存异常页面
var path = Path.Combine(WorkPath.ErrorPagePath, $"{ DateTime.Now.ToString("yyyy-MM-dd_HH.mm.ss.fffffff")}.html");
File.WriteAllText(path, dresp.Text);
return;
}
storeQueue.Post(dc);
}