public void TestTransformManyBlockConstructor()
{
// IEnumerable without option
var block = new TransformManyBlock<int, string>(i => new string[10]);
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
// Task without option
block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10]));
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
// IEnumerable with not cancelled token and default scheduler
block = new TransformManyBlock<int, string>(i => new string[10], new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
// Task with not cancelled token and default scheduler
block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10]), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
// IEnumerable with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new TransformManyBlock<int, string>(i => new string[10], new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
// Task with a cancelled token and default scheduler
token = new CancellationToken(true);
block = new TransformManyBlock<int, string>(i => Task.Factory.StartNew(() => (IEnumerable<string>)new string[10]), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformManyBlock.");
}
internal EvalOperation(OnlineTrainerSettingsInternal settings)
{
this.telemetry = new TelemetryClient();
// evaluation pipeline
this.evalEventHubClient = EventHubClient.CreateFromConnectionString(settings.EvalEventHubConnectionString);
this.evalBlock = new TransformManyBlock<object, EvalData>(
(Func<object, IEnumerable<EvalData>>)this.OfflineEvaluate,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 1024
});
this.evalBlock.Completion.ContinueWith(t =>
{
this.telemetry.TrackTrace($"Stage 3 - Evaluation pipeline completed: {t.Status}");
if (t.IsFaulted)
this.telemetry.TrackException(t.Exception);
});
// batch output together to match EventHub throughput by maintaining maximum latency of 5 seconds
this.evalBlockDisposable = this.evalBlock.AsObservable()
.GroupBy(k => k.PolicyName)
.Select(g =>
g.Window(TimeSpan.FromSeconds(5))
.Select(w => w.Buffer(245 * 1024, e => Encoding.UTF8.GetByteCount(e.JSON)))
.SelectMany(w => w)
.Subscribe(this.UploadEvaluation))
.Publish()
.Connect();
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
private static TransformManyBlock<int, int> ConstructTransformManyWithNMessages(int messagesCount)
{
var block = new TransformManyBlock<int, int>(i => new int[] { i });
for (int i = 0; i < messagesCount; i++)
{
block.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return block;
}
public void NullResultTest ()
{
bool received = false;
var transformMany =
new TransformManyBlock<int, int> (i => (IEnumerable<int>)null);
var action = new ActionBlock<int> (i => received = true);
transformMany.LinkTo (action);
Assert.IsTrue (transformMany.Post (1));
transformMany.Complete ();
Assert.IsTrue (transformMany.Completion.Wait (100));
Assert.IsFalse (received);
}
public PlayerMatchProducer(RiotApi riotApi, RiotQuerySettings querySettings, List<Queue> queryQueues, MatchDownloadLimiter testSynchronizer)
{
api = riotApi;
this.querySettings = querySettings;
this.queryQueues = queryQueues;
this.testSynchronizer = testSynchronizer;
// Create blocks
PlayerBufferBlock = new BufferBlock<PlayerEntry>();
PlayerToMatchesBlock = new TransformManyBlock<PlayerEntry, MatchSummary>(
async player => await ConsumePlayerAsync(player),
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 2 });
// Link blocks
PlayerBufferBlock.LinkTo(PlayerToMatchesBlock, new DataflowLinkOptions() { PropagateCompletion = true });
}
public async Task TestAutoComplete2()
{
var block2 = new TransformManyBlock<Int, Int>(i =>
{
return Enumerable.Empty<Int>();
});
var dataflow1 = DataflowUtils.FromDelegate<Int, Int>(i => new[] { i }); //preserve the guid
var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int() { Val = 1 });
Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(2)));
Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete"));
}
public void DeferredUsageTest ()
{
int insIndex = -1;
int[] array = new int[5 + 3];
var evt = new CountdownEvent (array.Length);
var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); });
var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i));
trsm.Post (5);
trsm.Post (3);
trsm.LinkTo (block);
evt.Wait ();
CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array);
}
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock()
{
var directoryBrowserBlock = new TransformManyBlock<string, string>(path =>
{
var dir = new DirectoryInfo(path);
return dir.EnumerateFileSystemInfos()
.Select(fi => fi.FullName);
});
var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName =>
{
var fi = new FileInfo(fileName);
return new KeyValuePair<string, long>(fileName, fi.Length);
});
directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists);
var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator);
return customBlock;
}
public async Task TestBlockBufferCount()
{
var block1 = new TransformBlock<int, int>(i => 2 * i);
var block2 = new TransformManyBlock<int, int>(i => new [] { i });
var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); });
block1.Post(0);
block2.Post(0);
block2.Post(0);
block3.Post(0);
block3.Post(0);
block3.Post(0);
await Task.Delay(200);
Assert.AreEqual(1, block1.GetBufferCount().Total());
Assert.AreEqual(2, block2.GetBufferCount().Total());
Assert.AreEqual(2, block3.GetBufferCount().Total());
}
public async Task TestAutoComplete1()
{
var block = new TransformManyBlock<Int, Int>(i =>
{
int j = i.Val + 1;
Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j);
Thread.Sleep(500);
if (j < 10) return new[] { new Int { Val = j } };
else return Enumerable.Empty<Int>();
});
var dataflow1 = DataflowUtils.FromDelegate<Int, Int>(i => i);
var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1));
dataflow1.GoTo(dataflow2).GoTo(dataflow1);
dataflow1.InputBlock.Post(new Int() { Val = 1 });
Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(10)));
}
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
public BreakAvailabilityCalculator(
ILogger logger,
ISqlServerDbContextFactory <ISqlServerLongRunningTenantDbContext> tenantDbContextFactory,
RecalculateBreakAvailabilityOptions recalculateOptions,
ITargetBlock <IBreakAvailability> targetBlock,
CancellationToken cancellationToken)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_tenantDbContextFactory = tenantDbContextFactory ?? throw new ArgumentNullException(nameof(tenantDbContextFactory));
_recalculateOptions = recalculateOptions ?? throw new ArgumentNullException(nameof(recalculateOptions));
_cancellationToken = cancellationToken;
_internalBlock = new TransformManyBlock <IBreakAvailability[], IBreakAvailability>(
breaks => breaks.AsEnumerable(), new ExecutionDataflowBlockOptions
{
CancellationToken = _cancellationToken
});
_ = _internalBlock.LinkTo(targetBlock ?? throw new ArgumentNullException(nameof(targetBlock)));
_ = _internalBlock.Completion.ContinueWith(_ => targetBlock.Complete());
_lockDbContext = new SemaphoreSlim(_recalculateOptions.BoundedDbContextOperationCapacity,
_recalculateOptions.BoundedDbContextOperationCapacity);
}
public async Task TestAutoComplete3()
{
var block1 = new TransformManyBlock <Int, Int>(i =>
{
return(new[] { i }); //preserve the guid
});
var block2 = new TransformManyBlock <Int, Int>(i =>
{
return(Enumerable.Empty <Int>());
});
var container1 = BlockContainerUtils.FromBlock(block1);
var container2 = BlockContainerUtils.AutoComplete(BlockContainerUtils.FromBlock(block2), TimeSpan.FromSeconds(1));
container1.LinkTo(container2);
container2.LinkTo(container1);
container1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(2)));
}
public void TransformManyOverfullTest()
{
var scheduler = new TestScheduler();
int n = 0;
var transform = new TransformManyBlock <int, int> (
i =>
{
Interlocked.Increment(ref n);
return(new[] { -i, i });
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 2, TaskScheduler = scheduler
});
Assert.IsTrue(transform.Post(1));
Assert.IsTrue(transform.Post(2));
Assert.GreaterOrEqual(scheduler.ExecuteAll(), 1);
Assert.AreEqual(2, Volatile.Read(ref n));
}
public LoopDataflow2()
{
var options = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 100
};
InputMessageBlock = new TransformBlock <Message, Message>(async msg => await InputMessage(msg));
HandleMessageBlock = new TransformManyBlock <Message, Message>(async msg => await HandleMessage(msg), options);
HandleMessageBlock2 = new TransformManyBlock <Message, Message>(async msg => await HandleMessage2(msg), options);
OutputMessageBlock = new ActionBlock <Message>(msg => OutputMessage(msg), options);
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = false
};
InputMessageBlock.LinkTo(HandleMessageBlock, linkOptions);
HandleMessageBlock.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true);
HandleMessageBlock.LinkTo(HandleMessageBlock2, linkOptions, msg => msg.WasProcessed == false);
HandleMessageBlock2.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true);
HandleMessageBlock2.LinkTo(HandleMessageBlock, linkOptions, msg => msg.WasProcessed == false);
InputMessageBlock.Completion.ContinueWith(async tsk => {
await BothMessageHandlersAreComplete();
HandleMessageBlock.Complete();
});
HandleMessageBlock.Completion.ContinueWith(tsk => {
HandleMessageBlock2.Complete();
});
HandleMessageBlock2.Completion.ContinueWith(tsk => {
OutputMessageBlock.Complete();
});
DebuggingLoop();
}
/// <summary>
/// 下载文件方法（自动确定是否使用分片下载）
/// </summary>
/// <param name="fileEnumerable">文件列表</param>
/// <param name="downloadParts"></param>
public static async Task AdvancedDownloadListFile(IEnumerable <DownloadFile> fileEnumerable,
int downloadParts = 16)
{
ProcessorHelper.SetMaxThreads();
var filesBlock =
new TransformManyBlock <IEnumerable <DownloadFile>, DownloadFile>(d =>
{
var dl = d.ToList();
foreach (var df in dl.Where(df => !Directory.Exists(df.DownloadPath)))
{
Directory.CreateDirectory(df.DownloadPath);
}
return(dl);
});
var actionBlock = new ActionBlock <DownloadFile>(async d =>
{
if (d.FileSize is >= 1048576 or 0)
{
await MultiPartDownloadTaskAsync(d, downloadParts);
}
public async Task TestAutoComplete2()
{
var block1 = new TransformManyBlock <Int, Int>(i =>
{
return(new[] { i }); //preserve the guid
});
var block2 = new TransformManyBlock <Int, Int>(i =>
{
int j = i.Val + 1;
Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j);
Thread.Sleep(500);
if (j < 10)
{
return new[] { new Int {
Val = j
} }
}
;
else
{
return(Enumerable.Empty <Int>());
}
});
var container1 = BlockContainerUtils.FromBlock(block1);
var container2 = BlockContainerUtils.AutoComplete(BlockContainerUtils.FromBlock(block2), TimeSpan.FromSeconds(1));
container1.LinkTo(container2);
container2.LinkTo(container1);
container1.InputBlock.Post(new Int()
{
Val = 1
});
Assert.IsTrue(await container2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(10)));
}
// TransformManyBlock is opposite to a BatchBlock. It takes a single input and creates many outputs.
// We will also be demonstrating how to link to an ActionBlock to work on those outputs.
// Also make note that the order is still ensured.
private static async Task SimpleDemoWithParallelismAsync()
{
Console.WriteLine("TransformManyBlockDemo has started!");
var transformManyBlock = new TransformManyBlock <int, string>(
ProduceTimeData,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 5
});
var actionBlock = new ActionBlock <string>(Console.WriteLine);
transformManyBlock.Post(20);
// Connect the source block (Transform) to the target/action block (Action).
transformManyBlock.LinkTo(actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
transformManyBlock.Complete();
await transformManyBlock.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
private ISourceBlock <RunningGame> GameProcessPoller(Config config, IEnumerable <IGameProcessInfo> gameProcessInfos)
{
Dictionary <string, string> gameByProcess = GameByProcess(gameProcessInfos);
Func <IEnumerable <string>, IEnumerable <string?> > keepOne = new KeepOne <string>(gameByProcess.Keys).Call;
var transformer = new TransformManyBlock <IEnumerable <string>, RunningGame>(
processNames => keepOne(processNames).Select(processName =>
{
if (processName == null)
{
_logger.LogInformation("The game has been closed");
return(new RunningGame(null));
}
var game = gameByProcess.GetValueOrDefault(processName);
_logger.LogInformation("Found game {string?} (process {string?})", game, processName);
return(new RunningGame(game));
})
);
var source = PollingSource.Create(config.PollingInterval, () => Process.GetProcesses().Select(p => p.ProcessName));
source.LinkTo(transformer);
transformer.Completion.ContinueWith(_ => source.Complete());
return(transformer);
}
/// <summary>
/// The following basic example creates a TransformManyBlock<TInput,TOutput> object
/// that splits strings into their individual character sequences. The TransformManyBlock<TInput,TOutput>
/// object takes String values as input and produces Char values as output.
/// </summary>
/// <param name="type"></param>
public static void Run(int type = 1)
{
// Create a TransformManyBlock<string, char> object that splits
// a string into its individual characters.
TransformManyBlock <string, char> transformManyBlock = new TransformManyBlock <string, char>(s =>
{
char[] charArray = s.ToCharArray();
return(charArray);
});
// Post two messages to the first block.
transformManyBlock.Post("Hello");
transformManyBlock.Post("World");
// Receive all output values from the block.
if (type == 0)
{
for (int i = 0; i < ("Hello" + "World").Length; i++)
{
Console.WriteLine(transformManyBlock.Receive());
}
}
else if (type == 1)
{
do
{
if (transformManyBlock.InputCount == 0 && transformManyBlock.OutputCount == 0)
{
return;
}
char receive = transformManyBlock.Receive();
Console.WriteLine("transformManyBlock.Receive:");
Console.WriteLine(receive);
Console.WriteLine();
} while (true);
}
}
public async Task TestCountZeroAtCompletion()
{
var cts = new CancellationTokenSource();
var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token
});
tb.Post(1);
cts.Cancel();
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
cts = new CancellationTokenSource();
tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(1);
((IDataflowBlock)tb).Fault(new InvalidOperationException());
await Assert.ThrowsAnyAsync <InvalidOperationException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
}
public async Task TestCircularLinkingAsyncEnumerable()
{
const int Iters = 200;
var tcs = new TaskCompletionSource <bool>();
IAsyncEnumerable <int> body(int i)
{
if (i >= Iters)
{
tcs.SetResult(true);
}
return(AsyncEnumerable.Repeat(i + 1, 1));
}
TransformManyBlock <int, int> tb = new TransformManyBlock <int, int>(body);
using (tb.LinkTo(tb))
{
tb.Post(0);
await tcs.Task;
tb.Complete();
}
}
private static void Main(string[] args)
{
var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num));
var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 });
var finishingBlock = new ActionBlock<string>(str =>
{
writerBlock.Completion.Wait();
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str);
});
generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true});
writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true });
for (var i = 1; i <= 3; i++)
{
Console.WriteLine("Posted " + i*10);
generatorBlock.Post(i*10);
}
generatorBlock.Complete();
finishingBlock.Completion.Wait();
Console.WriteLine("Pipeline is finished");
Console.ReadKey();
}
public async Task TestYieldingNoResults()
{
foreach (int dop in new[] { 1, Environment.ProcessorCount })
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
const int Modes = 3, Iters = 100;
var tb = new TransformManyBlock<int, int>(i => {
switch (i % Modes)
{
default:
case 0:
return new List<int> { i };
case 1:
return new int[0];
case 2:
return new Collection<int> { i, i + 1 };
}
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = boundedCapacity });
var source = new BufferBlock<int>();
source.PostRange(0, Modes * Iters);
source.Complete();
source.LinkTo(tb, new DataflowLinkOptions { PropagateCompletion = true });
int received = 0;
while (await tb.OutputAvailableAsync())
{
await tb.ReceiveAsync();
received++;
}
Assert.Equal(expected: Modes * Iters, actual: received);
}
}
private static IDictionary <string, uint> GetTopWordsDataFlow()
{
Console.WriteLine(nameof(GetTopWordsDataFlow) + "...");
const int WorkerCount = 12;
var result = new ConcurrentDictionary <string, uint>(StringComparer.InvariantCultureIgnoreCase);
const int BoundedCapacity = 10000;
var bufferBlock = new BufferBlock <string>(
new DataflowBlockOptions {
BoundedCapacity = BoundedCapacity
});
var splitLineToWordsBlock = new TransformManyBlock <string, string>(
line => line.Split(Separators, StringSplitOptions.RemoveEmptyEntries),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = BoundedCapacity
});
var batchWordsBlock = new BatchBlock <string>(5000);
var trackWordsOccurrencBlock = new ActionBlock <string[]>(words =>
{
foreach (var word in words)
{
if (!IsValidWord(word))
{
continue;
}
result.AddOrUpdate(word, 1, (key, oldVal) => oldVal + 1);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = WorkerCount
});
var defaultLinkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
bufferBlock.LinkTo(splitLineToWordsBlock, defaultLinkOptions);
splitLineToWordsBlock.LinkTo(batchWordsBlock, defaultLinkOptions);
batchWordsBlock.LinkTo(trackWordsOccurrencBlock, defaultLinkOptions);
// Begin producing
foreach (var line in File.ReadLines(InputFile.FullName))
{
bufferBlock.SendAsync(line).Wait();
}
bufferBlock.Complete();
// End of producing
// Wait for workers to finish their work
trackWordsOccurrencBlock.Completion.Wait();
return(result
.OrderByDescending(kv => kv.Value)
.Take((int)TopCount)
.ToDictionary(kv => kv.Key, kv => kv.Value));
}
public void AsyncNullTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformManyBlock<int, int> (
i => (Task<IEnumerable<int>>)null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
private Tuple <ITargetBlock <Assembly>, Task <DiscoveredParts> > CreateAssemblyDiscoveryBlockChain(IProgress <DiscoveryProgress>?progress, CancellationToken cancellationToken)
{
var progressFilter = new ProgressFilter(progress);
var tuple = this.CreateDiscoveryBlockChain(false, progressFilter, cancellationToken);
var exceptions = new List <PartDiscoveryException>();
var assemblyBlock = new TransformManyBlock <Assembly, Type>(
a =>
{
IReadOnlyCollection <Type> types;
try
{
// Fully realize any enumerable now so that we can catch the exception rather than
// leave it to dataflow to catch it.
types = this.GetTypes(a).ToList();
}
catch (ReflectionTypeLoadException ex)
{
var partDiscoveryException = new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.ReflectionTypeLoadExceptionWhileEnumeratingTypes, a.Location), ex)
{
AssemblyPath = a.Location
};
lock (exceptions)
{
exceptions.Add(partDiscoveryException);
}
types = ex.Types.Where(t => t != null).ToList();
}
catch (Exception ex)
{
var partDiscoveryException = new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToEnumerateTypes, a.Location), ex)
{
AssemblyPath = a.Location
};
lock (exceptions)
{
exceptions.Add(partDiscoveryException);
}
return(Enumerable.Empty <Type>());
}
progressFilter.OnDiscoveredMoreTypes(types.Count);
return(types);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Debugger.IsAttached ? 1 : Environment.ProcessorCount,
CancellationToken = cancellationToken,
});
assemblyBlock.LinkTo(tuple.Item1, new DataflowLinkOptions {
PropagateCompletion = true
});
var tcs = new TaskCompletionSource <DiscoveredParts>();
Task.Run(async delegate
{
try
{
var parts = await tuple.Item2.ConfigureAwait(false);
tcs.SetResult(parts.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions)));
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
return(Tuple.Create <ITargetBlock <Assembly>, Task <DiscoveredParts> >(assemblyBlock, tcs.Task));
}
static void Main()
{
var sw = new Stopwatch();
sw.Start();
var loadSourceFileToMemory = new TransformBlock <string, string>(async path =>
{
Console.WriteLine("Loading to memory '{0}'...", path);
using (StreamReader SourceReader = File.OpenText(path))
{
return(await SourceReader.ReadToEndAsync());
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelismLoad
});
var generateTestClass = new TransformManyBlock <string, TestClass>(async text =>
{
Console.WriteLine("Generating test classes...");
TestClassGenerator classGenerator = new TestClassGenerator();
return(classGenerator.GenerateTestClasses(text));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelismGenerate
});
var saveTestClassToFile = new ActionBlock <TestClass>(async testClass =>
{
using (StreamWriter DestinationWriter = File.CreateText(DestPath + testClass.FileName))
{
Console.WriteLine("Saving '{0}' on disk...", testClass.FileName);
await DestinationWriter.WriteAsync(testClass.Source);
Console.WriteLine("Saved '{0}'!", testClass.FileName);
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelismSave
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
loadSourceFileToMemory.LinkTo(generateTestClass, linkOptions);
generateTestClass.LinkTo(saveTestClassToFile, linkOptions);
foreach (var path in _input)
{
loadSourceFileToMemory.Post(path);
}
loadSourceFileToMemory.Complete();
saveTestClassToFile.Completion.Wait();
sw.Stop();
Console.WriteLine(sw.ElapsedMilliseconds);
}
public void Test()
{
var action = new ActionBlock <int[]>(x => { }, new ExecutionDataflowBlockOptions {
});
var batch = new BatchBlock <int>(10, new GroupingDataflowBlockOptions {
});
using var likBA = batch.LinkTo(action, new DataflowLinkOptions { }, x => x[0] > 1); //actionBlock gets 10 elements
batch.TriggerBatch(); //for leftover elements
var batchJoined = new BatchedJoinBlock <int[], string>(10, new GroupingDataflowBlockOptions {
});
batch.LinkTo(batchJoined.Target1, new DataflowLinkOptions {
}, ints => ints[0] > 0);
var action2 = new ActionBlock <Tuple <IList <int[]>, IList <string> > >(x => { });
using var linkBJA = batchJoined.LinkTo(action2, new DataflowLinkOptions { }, x => x.Item1.Count == x.Item2.Count);//it's 10 total so 7-3 or 0-10 will propagate
var broadcast = new BroadcastBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
});
var action3 = new ActionBlock <int>(x => { });
var action4 = new ActionBlock <int>(x => { });
broadcast.LinkTo(action3, new DataflowLinkOptions {
}, x => x > 0);
broadcast.LinkTo(action4, new DataflowLinkOptions {
}, x => x > 0); //both will gete same elements
var buffer = new BufferBlock <int>(new DataflowBlockOptions {
});
var action5 = new ActionBlock <int>(x => { });
buffer.LinkTo(action5, new DataflowLinkOptions {
}, x => x == 0);
var buffer1 = new BufferBlock <int>();
var join = new JoinBlock <int, string>(new GroupingDataflowBlockOptions {
});
buffer1.LinkTo(join.Target1);
var action6 = new ActionBlock <Tuple <int, string> >(tlp => { });
join.LinkTo(action6);
var transform = new TransformBlock <int, string>(x => x.ToString(), new ExecutionDataflowBlockOptions {
});
var action7 = new ActionBlock <string>(str => { });
transform.LinkTo(action7);
var transformMany = new TransformManyBlock <int, string>(
x => Enumerable.Range(0, x).Select(i => i.ToString()));//one recived to many output
var action8 = new ActionBlock <string>(str => { });
var writeOnce = new WriteOnceBlock <int>(cloningFunction: x => x, new DataflowBlockOptions {
}); // it gets and stores 1 element but gives clones to linked blocks?
var action9 = new ActionBlock <int>(x => { });
writeOnce.LinkTo(action9);
var bufferC1 = new BufferBlock <int>();
var bufferC2 = new BufferBlock <string>();
var actionIdx = DataflowBlock.Choose <int, string>(
bufferC1, x => Console.WriteLine(x + 1),
bufferC2, str => Console.WriteLine(str + ""));
var actionE1 = new ActionBlock <int>(x => { });
var bufferE1 = new BufferBlock <string>();
var propagator = DataflowBlock.Encapsulate <int, string>(actionE1, bufferE1);
var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount, int.MaxValue);
var concurrent = scheduler.ConcurrentScheduler;
var exclusive = scheduler.ExclusiveScheduler;
}
public void BuildPipeline(CancellationToken token)
{
// create the blocks
getFileUrls = new TransformManyBlock<string, string>(
(url) =>
{
List<string> fileUrls = new List<string>();
GetFileUrls(url, fileUrls);
return fileUrls;
}
, new ExecutionDataflowBlockOptions { CancellationToken = token }
);
var downloadFile = new TransformBlock<string, string>(
(fileUrl) =>
{
string fileName = Regex.Match(fileUrl, @"([^/]+$)").Value;
using (var client = new WebClient())
{
client.DownloadFile(fileUrl, FolderPath + "\\" + fileName);
}
Interlocked.Increment(ref currentFileCount);
UpdateProgress((double)currentFileCount / (double)fileCount);
return FolderPath + "\\" + fileName;
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token,
}
);
var convertFile = new TransformBlock<string, string>(
(fileName) =>
{
string newFileName = ConvertedFilesFolder + @"\" + Regex.Match(fileName, @"([^\\]+$)").Value + "x";
Process process = new Process();
process.StartInfo.Arguments = string.Format(@" -nme -oice {0} {1}", fileName, newFileName);
process.StartInfo.FileName = @"c:\Program Files (x86)\Microsoft Office\Office12\excelcnv.exe";
process.Start();
while (!process.WaitForExit(15000))
{
process.Kill();
process.Start();
}
Interlocked.Increment(ref convertedFilesCount);
UpdateConversionProgress((double)convertedFilesCount / (double)fileCount);
return newFileName;
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token
}
);
var parseCitizenInfo = new ActionBlock<string>(
(fileName) =>
{
var referendumProcessor = new ReferendumProcessor();
citizensOfArmenia.Post(referendumProcessor.ProcessFile(fileName));
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token
}
);
// Link the blocks
citizensOfArmenia = new BufferBlock<List<Citizen>>();
getFileUrls.LinkTo(downloadFile, new DataflowLinkOptions { PropagateCompletion = true });
downloadFile.LinkTo(convertFile, new DataflowLinkOptions { PropagateCompletion = true });
convertFile.LinkTo(parseCitizenInfo, new DataflowLinkOptions { PropagateCompletion = true });
parseCitizenInfo.Completion.ContinueWith((tsk) => citizensOfArmenia.Complete());
}
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt");
bathymetry.ToYXZ(yxzFileName, -1);
var mbs = new C3mbs();
mbsRESULT mbsResult;
if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath())))
throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult));
var config = mbs.GetConfiguration();
config.enabled = false; // binary output enabled/disabled
config.durationLess = true; // make sure we're in durationless mode.
mbs.SetConfiguration(config);
mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath));
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
mbsRESULT retval;
lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 });
if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
//mbsResult = mbs.InitializeRun();
//if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1);
//else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.FinishRun();
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult));
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
public static IPropagatorBlock<FileWord, FileWord> GetFileWordSaverBlock(Func<IRepository> repositoryFactory)
{
var inputBuffer = new BufferBlock<FileWord>();
var outputBuffer = new BufferBlock<FileWord>();
var batchBlockWithoutParents = new BatchBlock<FileWord>(1000);
inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0);
inputBuffer.PropagateCompleted(batchBlockWithoutParents);
var saveWithoutParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
return await SaveEntities(repositoryFactory, fileWords, true);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock);
saveWithoutParentsBlock.LinkTo(outputBuffer);
var batchBlockWithParents = new BatchBlock<FileWord>(300);
// MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages.
// This is required to prohibit "stealing" of workload from saveWithoutParentsBlock
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
inputBuffer.PropagateCompleted(batchBlockWithParents);
var saveWithParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
var res = await SaveEntities(repositoryFactory, fileWords, false);
FileWord fileWord;
if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents
{
if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0)
{
batchBlockWithoutParents.Post(fileWord);
}
else
{
batchBlockWithParents.Post(fileWord);
}
}
// Link again for another 300 messages
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
return res;
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted
});
batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock);
saveWithParentsBlock.LinkTo(outputBuffer);
saveWithoutParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
saveWithParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion)
.ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted);
return DataflowBlock.Encapsulate(inputBuffer, outputBuffer);
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
return Enumerable.Range(0, 10);
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
if (i < ITERS - 1) throw new OperationCanceledException();
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return null;
}
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
}), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
return Task.Factory.StartNew(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
});
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
var cts = new CancellationTokenSource();
return Task.Factory.StartNew(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, cts.Token);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer)
{
Requires.NotNull(inboxContainer, "inboxContainer");
var deleteBlobsExpiringBefore = DateTime.UtcNow;
int purgedBlobCount = 0;
var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>(
async c =>
{
try
{
var results = await c.ListBlobsSegmentedAsync(
string.Empty,
useFlatBlobListing: true,
pageSize: 50,
details: BlobListingDetails.Metadata,
options: new BlobRequestOptions(),
operationContext: null);
return from blob in results.OfType<ICloudBlob>()
let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey])
where expires < deleteBlobsExpiringBefore
select blob;
}
catch (StorageException ex)
{
var webException = ex.InnerException as WebException;
if (webException != null)
{
var httpResponse = (HttpWebResponse)webException.Response;
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
// it's legit that some tests never created the container to begin with.
return Enumerable.Empty<ICloudBlob>();
}
}
throw;
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<ICloudBlob>(
blob =>
{
Interlocked.Increment(ref purgedBlobCount);
return blob.DeleteAsync();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredBlobs.Post(inboxContainer);
searchExpiredBlobs.Complete();
await deleteBlobBlock.Completion;
}
public async Task TransformManyEnumerableToAction()
{
var data = new[] { 1 };
var tm = new TransformManyBlock<int, int>(i => data);
int completedCount = 0;
var c = new ActionBlock<int>(i => completedCount++);
tm.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
tm.PostRange(0, Iterations);
tm.Complete();
await c.Completion;
Assert.Equal(expected: Iterations, actual: completedCount);
}
internal static void Run()
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words
.Where(word => word.Length > 3)
.Distinct()
.ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Dangerous to use with PLINQ
var wordsSet = new HashSet <string>(words);
return(from word in words.AsParallel()
let reverse = new string(word.Reverse().ToArray())
where word != reverse && wordsSet.Contains(reverse) &&
wordsSet.Remove(word) && wordsSet.Remove(reverse)
orderby word
select word);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(createWordList, linkOptions);
createWordList.LinkTo(filterWordList, linkOptions);
filterWordList.LinkTo(findReversedWords, linkOptions);
findReversedWords.LinkTo(printReversedWords, linkOptions);
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Process The Bible
downloadString.Post("https://www.gutenberg.org/files/30/30.txt");
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
}
public SimpleLogReader(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
this.m_parsingBlock = new TransformManyBlock<string, string>(
s =>
{
if (string.IsNullOrEmpty(s))
{
return Enumerable.Empty<string>();
}
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
return s.Split(Splitor);
});
this.m_recorder = new StatisticsRecorder(this);
this.m_recordBlock = new ActionBlock<string>(
s =>
{
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
this.m_recorder.RecordEvent(s);
});
var df1 = DataflowUtils.FromBlock(m_parsingBlock);
var df2 = DataflowUtils.FromBlock(m_recordBlock);
df1.LinkTo(df2);
RegisterChild(df1);
RegisterChild(df2);
}
static void Main(string[] args)
{
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading {0}...{1} #{2}", uri, DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId);
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
var createWordlist = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Create word list...{0} #{1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId);
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] token = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(token);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
var filterWordlist = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filter word list...{0} #{1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId);
return(words.Where(word => word.Length > 3)
.Distinct().ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReverseWordlist = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Find reversed word list...{0} {1}", DateTime.Now.ToString("HH:mm:ss fff"), Thread.CurrentThread.ManagedThreadId);
var wordset = new HashSet <string>(words);
return(from word in words.AsParallel()
let reversed = new string(word.Reverse().ToArray())
where word != reversed && wordset.Contains(reversed)
select word);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversed =>
{
Console.WriteLine("Found reversed word: {0}/{1} #{2} {3}", reversed, new string(reversed.Reverse().ToArray()),
Thread.CurrentThread.ManagedThreadId, DateTime.Now.ToString("HH:mm:ss fff"));
});
//
// Connect the dataflow blocks to form a pipeline.
//
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(createWordlist, linkOptions);
createWordlist.LinkTo(filterWordlist, linkOptions);
filterWordlist.LinkTo(findReverseWordlist, linkOptions);
findReverseWordlist.LinkTo(printReversedWords, linkOptions);
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.WriteLine("Total thread {0}.", System.Diagnostics.Process.GetCurrentProcess().Threads.Count);
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock<string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return new WebClient().DownloadString(uri);
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock<string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
tokens[i] = ' ';
}
text = new string(tokens);
// Separate the text into an array of words.
return text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries);
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock<string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock<string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue<string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch<string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return reversedWords;
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock<string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception);
else createWordList.Complete();
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception);
else filterWordList.Complete();
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception);
else findReversedWords.Complete();
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception);
else printReversedWords.Complete();
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
}
[InlineData(2, 1, false)] // no force ordered, but dop == 1, so it doesn't matter
public async Task TestOrdering_Async_OrderedEnabled(int mmpt, int dop, bool? EnsureOrdered)
{
const int iters = 1000;
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt };
if (EnsureOrdered == null)
{
Assert.True(options.EnsureOrdered);
}
else
{
options.EnsureOrdered = EnsureOrdered.Value;
}
var tb = new TransformManyBlock<int, int>(i => Task.FromResult(Enumerable.Repeat(i, 1)), options);
tb.PostRange(0, iters);
for (int i = 0; i < iters; i++)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration)
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false };
var mres = new ManualResetEventSlim();
var tb = new TransformManyBlock<int, int>(i =>
{
if (i == 0) mres.Wait();
return trustedEnumeration ?
new[] { i } :
Enumerable.Repeat(i, 1);
}, options);
tb.Post(0);
tb.Post(1);
Assert.Equal(1, await tb.ReceiveAsync());
mres.Set();
Assert.Equal(0, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
}
static void Main(string[] args)
{
StringComparison comparison = StringComparison.InvariantCultureIgnoreCase;
Predicate <string> linkFilter = link =>
link.IndexOf(".aspx", comparison) != -1 ||
link.IndexOf(".php", comparison) != -1 ||
link.IndexOf(".htm", comparison) != -1 ||
link.IndexOf(".html", comparison) != -1 ||
link.EndsWith(".com", comparison) ||
link.EndsWith(".net", comparison);
Predicate <string> imgFilter = url =>
url.EndsWith(".jpg", comparison) ||
url.EndsWith(".png", comparison) ||
url.EndsWith(".gif", comparison);
var downloader = new TransformBlock <string, string>(
async(url) =>
{
// using IOCP the thread pool worker thread does return to the pool
var client = new HttpClient();
string result = await client.GetStringAsync(url);
return(result);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 2
});
var contentBroadcaster = new BroadcastBlock <string>(s => s);
var linkParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var links = _linkRegexHRef.Matches(html);
foreach (Match item in links)
{
var value = item.Groups["LINK"].Value;
output.Add(value);
}
return(output);
});
var imgParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var images = _imgRegex.Matches(html);
foreach (Match item in images)
{
var value = item.Groups["IMG"].Value;
output.Add(value);
}
return(output);
});
var writer = new ActionBlock <string>(async url =>
{
var client = new HttpClient();
// using IOCP the thread pool worker thread does return to the pool
byte[] buffer = await client.GetByteArrayAsync(url);
string fileName = Path.GetFileName(url);
Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"\Images\");
string name = AppDomain.CurrentDomain.BaseDirectory + @"\Images\" + fileName;
using (Stream srm = File.OpenWrite(name))
{
await srm.WriteAsync(buffer, 0, buffer.Length);
}
});
var linkBroadcaster = new BroadcastBlock <string>(s => s);
IDisposable disposeAll = new CompositeDisposable(
// from [downloader] to [contentBroadcaster]
downloader.LinkTo(contentBroadcaster),
// from [contentBroadcaster] to [imgParser]
contentBroadcaster.LinkTo(imgParser),
// from [contentBroadcaster] to [linkParserHRef]
contentBroadcaster.LinkTo(linkParser),
// from [linkParser] to [linkBroadcaster]
linkParser.LinkTo(linkBroadcaster),
// conditional link to from [linkBroadcaster] to [downloader]
linkBroadcaster.LinkTo(downloader, linkFilter),
// from [linkBroadcaster] to [writer]
linkBroadcaster.LinkTo(writer, imgFilter),
// from [imgParser] to [writer]
imgParser.LinkTo(writer));
downloader.Post("http://www.reactivex.io");
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
downloader.Complete();
}
private static void DownloadBuilds(Args arguments)
{
s_logger.Info("Downloading Builds...");
// a couple of checks here
Contract.Requires(arguments.NumBuilds > 0, "You must specify a number of builds");
Contract.Requires(arguments.Year > 0, "You must specify a year");
Contract.Requires(arguments.Month > 0, "You must specify a month");
Contract.Requires(arguments.Day > 0, "You must specify a day");
Contract.Requires(arguments.AppConfig != null, "You must specify a configuration file");
Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory");
Contract.Requires(File.Exists(arguments.AppConfig.AnalyzerConfig.Exe), "The analyzer executable file must exist");
Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist");
// so in here we will create a new network.
var buildInfoBlock = new TransformManyBlock <GetKustoBuildInput, List <KustoBuild> >(i =>
{
var action = new GetKustoBuild(arguments.AppConfig);
var result = action.PerformAction(i);
// its not worth it to continue if this fails
if (!result.ExecutionStatus)
{
s_logger.Error(result.Exception, "Could not download builds");
throw result.Exception;
}
return(result.Result.KustoBuildData);
});
var downloadBlock = new TransformBlock <List <KustoBuild>, TimedActionResult <DecompressionOutput> >(i =>
{
var action = new BuildDownload(arguments.AppConfig, arguments.OutputDirectory);
return(action.PerformAction(i));
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxDownloadTasks
}
);
var analysisBlock = new ActionBlock <TimedActionResult <DecompressionOutput> >(i =>
{
// check
if (i.ExecutionStatus)
{
// analyze if decompression succeded
var action = new BuildAnalisys(arguments.AppConfig);
action.PerformAction(i.Result);
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxAnalysisTasks
}
);
// link them
buildInfoBlock.LinkTo(downloadBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
downloadBlock.LinkTo(analysisBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
// in here, we need to set up the days for which the downloads will be taken
var date = new DateTime(arguments.Year, arguments.Month, arguments.Day);
for (var i = 0; i < arguments.Span; ++i)
{
s_logger.Info($"Downloading builds from year={date.Year}, month={date.Month}, day={date.Day}");
// create the inout for this task
var input = new GetKustoBuildInput(arguments.NumBuilds, date.Year, date.Month, date.Day);
// post the task...
buildInfoBlock.Post(input);
// and add one more day
date = date.AddDays(1);
}
// and complete
buildInfoBlock.Complete();
// wait for the last...
analysisBlock.Completion.Wait();
}
public async Task Process(CancellationToken cancellationToken)
{
DataflowLinkOptions linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
DataflowLinkOptions nonPropagatingLinkOptions = new DataflowLinkOptions();
ExecutionDataflowBlockOptions singleItemBlockOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 1, EnsureOrdered = false
};
ExecutionDataflowBlockOptions bufferBlockOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 100, EnsureOrdered = false
};
ExecutionDataflowBlockOptions unboundedBlockOptions = new ExecutionDataflowBlockOptions {
EnsureOrdered = false
};
GroupingDataflowBlockOptions batchingBlockOptions = new GroupingDataflowBlockOptions {
EnsureOrdered = false
};
TransformManyBlock <object, MxHostTestDetails> queuePoller =
new TransformManyBlock <object, MxHostTestDetails>(_ => GetMxHostToProcess(_), singleItemBlockOptions);
TransformBlock <MxHostTestDetails, MxHostTestDetails> retestPeriodFilter =
new TransformBlock <MxHostTestDetails, MxHostTestDetails>((Func <MxHostTestDetails, MxHostTestDetails>)MarkTestToSkip, singleItemBlockOptions);
BufferBlock <MxHostTestDetails> buffer = new BufferBlock <MxHostTestDetails>(bufferBlockOptions);
TransformManyBlock <MxHostTestDetails, MxHostTestDetails> duplicateFilter =
new TransformManyBlock <MxHostTestDetails, MxHostTestDetails>(_ => FilterHosts(_), singleItemBlockOptions);
List <TransformBlock <MxHostTestDetails, MxHostTestDetails> > mxTestProcessors = Enumerable
.Range(1, _config.TlsTesterThreadCount)
.Select(index => new TransformBlock <MxHostTestDetails, MxHostTestDetails>(CreateTlsTester(Guid.NewGuid()), singleItemBlockOptions))
.ToList();
BatchBlock <MxHostTestDetails> resultBatcher =
new BatchBlock <MxHostTestDetails>(_config.PublishBatchSize, batchingBlockOptions);
Timer timer = new Timer(_ =>
{
resultBatcher.TriggerBatch();
_log.LogDebug("Batch triggered.");
});
TransformBlock <MxHostTestDetails, MxHostTestDetails> batchFlusher =
new TransformBlock <MxHostTestDetails, MxHostTestDetails>(ResetTimer(timer), unboundedBlockOptions);
TransformBlock <MxHostTestDetails[], MxHostTestDetails[]> resultPublisher =
new TransformBlock <MxHostTestDetails[], MxHostTestDetails[]>(_ => PublishResults(_), unboundedBlockOptions);
ActionBlock <MxHostTestDetails[]> deleteFromQueue =
new ActionBlock <MxHostTestDetails[]>(_ => RemoveFromQueue(_), unboundedBlockOptions);
queuePoller.LinkTo(retestPeriodFilter, linkOptions);
retestPeriodFilter.LinkTo(batchFlusher, nonPropagatingLinkOptions, x => x.SkipTesting);
retestPeriodFilter.LinkTo(buffer, linkOptions, x => !x.SkipTesting);
buffer.LinkTo(duplicateFilter, linkOptions);
mxTestProcessors.ForEach(processor => {
duplicateFilter.LinkTo(processor, linkOptions);
processor.LinkTo(batchFlusher, nonPropagatingLinkOptions);
});
batchFlusher.LinkTo(resultBatcher, linkOptions);
resultBatcher.LinkTo(resultPublisher, linkOptions);
resultPublisher.LinkTo(deleteFromQueue, linkOptions);
var blocks = new Dictionary <string, Func <int> > {
["Queued"] = () => queuePoller.OutputCount + buffer.Count,
["Processing"] = () => _processingFilter.HostCount,
};
var processorTasks = mxTestProcessors.Select(processor => processor.Completion).ToArray();
// Start the stats print loop
var statsTask = PrintStats(blocks, cancellationToken);
await RunPipeline(queuePoller, cancellationToken);
_log.LogInformation("Shutting down TLS Tester");
queuePoller.Complete();
_log.LogInformation("Waiting for test processors to complete...");
await Task.WhenAll(processorTasks);
_log.LogInformation("Test processors complete. Flushing results...");
batchFlusher.Complete();
_log.LogInformation("Waiting for results flush and final shutdown...");
await Task.WhenAll(
statsTask,
deleteFromQueue.Completion
);
_log.LogInformation("TLS tester shut down. Exiting.");
}
private static async Task <int> Main()
{
string?knownArtistListFilename = Environment.GetCommandLineArgs().Skip(1).FirstOrDefault();
if (knownArtistListFilename == null)
{
Console.WriteLine("Please pass the filename of a newline-delimited text file containing a list of known artists to look up as the first argument.");
return(1);
}
ISet <string> knownArtistNames = new HashSet <string>((await File.ReadAllLinesAsync(knownArtistListFilename, Encoding.UTF8)).Select(s => s.ToLowerInvariant()));
var blockOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = MAX_PARALLEL_DOWNLOADS
};
TransformBlock <string, ArtistPage> fetchSource = new TransformBlock <string, ArtistPage>(async artistName => {
Uri artistUri = FluentUriBuilder.From(BASE_URI.AbsoluteUri).Path(artistName.ToLowerInvariant()).ToUri();
using HttpResponseMessage response = await CLIENT.GetAsync(artistUri);
var artistPage = new ArtistPage {
artistName = artistName
};
if (response.IsSuccessStatusCode)
{
artistPage.sourceCode = await response.Content.ReadAsStringAsync();
}
else
{
Console.WriteLine($"Error: Failed to download artist page for {artistName}: {response.StatusCode}. Please check the artist name on {BASE_URI}");
}
return(artistPage);
}, blockOptions);
TransformManyBlock <ArtistPage, Relationship> extractRelationships = new TransformManyBlock <ArtistPage, Relationship>(page => {
if (page.sourceCode == null)
{
return(null);
}
MatchCollection artistNameMatches = Regex.Matches(page.sourceCode, @"<a href=""(?<artistSlug>.+?)"" class=S id=s\d+>(?<artistName>.+?)</a>");
IEnumerable <string> relatedArtistNames = artistNameMatches.Skip(1).Select(match => match.Groups["artistName"].Value);
Match relationshipStrengthMatches = Regex.Match(page.sourceCode, @"Aid\[0\]=new Array\(-1(?:,(?<strength>-?\d+(?:\.\d+)?))*\);");
IEnumerable <double> relatedArtistStrengths = relationshipStrengthMatches.Groups["strength"].Captures.Select(capture => double.Parse(capture.Value));
return(relatedArtistNames.Zip(relatedArtistStrengths)
.Select(tuple => new Relationship {
knownArtist = page.artistName, unknownArtist = tuple.First, strength = tuple.Second
}));
}, blockOptions);
IDictionary <string, double> strengthsByArtistName = new Dictionary <string, double>();
ActionBlock <Relationship> incoming = new ActionBlock <Relationship>(relationship => {
if (knownArtistNames.Contains(relationship.unknownArtist.ToLowerInvariant()))
{
return;
}
strengthsByArtistName.TryGetValue(relationship.unknownArtist, out double existingStrength);
strengthsByArtistName[relationship.unknownArtist] = existingStrength + relationship.strength;
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
var outgoing = new BufferBlock <IDictionary <string, double> >();
#pragma warning disable 4014
incoming.Completion.ContinueWith(task => { outgoing.Post(strengthsByArtistName); });
#pragma warning restore 4014
IPropagatorBlock <Relationship, IDictionary <string, double> > reduce = DataflowBlock.Encapsulate(incoming, outgoing);
ActionBlock <IDictionary <string, double> > printReport = new ActionBlock <IDictionary <string, double> >(strengths => {
IOrderedEnumerable <KeyValuePair <string, double> > sorted = strengths.OrderByDescending(pair => pair.Value);
foreach ((string unknownArtistName, double strength) in sorted)
{
Console.WriteLine($"{strength,5:N1}\t{unknownArtistName}");
}
});
DataflowLinkOptions linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
fetchSource.LinkTo(extractRelationships, linkOptions);
extractRelationships.LinkTo(reduce, linkOptions);
reduce.LinkTo(printReport, linkOptions);
foreach (string knownArtistName in knownArtistNames)
{
fetchSource.Post(knownArtistName);
}
fetchSource.Complete();
await printReport.Completion;
return(0);
}
public void ListWords()
{
// Create the members of the pipeline.
Console.WriteLine("start");
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(uri => downLoadStringFunc(uri));
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text => createWordListFunc(text));
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words => filterWorldListFunc(words));
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue <string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch <string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return(reversedWords);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord => printReversedWordsFunc(reversedWord));
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)findReversedWords).Fault(t.Exception);
}
else
{
findReversedWords.Complete();
}
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printReversedWords).Fault(t.Exception);
}
else
{
printReversedWords.Complete();
}
});
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
downloadString.Complete();
printReversedWords.Completion.Wait();
Console.Read();
}
internal static bool TransformManyEnumerableToAction()
{
const int ITERS = 2;
var data = new[] { 1 };
var tm = new TransformManyBlock<int, int>(i => data);
int completedCount = 0;
var c = new ActionBlock<int>(i => completedCount++);
tm.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++) tm.Post(i);
tm.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
private static void DownloadMonthlyQueueData(Args arguments)
{
s_logger.Info($"Downloading Queue data ({(arguments.IncludeMachineMap? "including machine map" : "no machine map will be included")})");
// couple of checks here
Contract.Requires(arguments.Year > 0, "You must specify a year");
Contract.Requires(arguments.Month > 0, "You must specify a month");
Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory");
Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist");
if (arguments.IncludeMachineMap)
{
Contract.Requires(arguments.AppConfig.ConcurrencyConfig.MaxMachineMapTasks > 0, "You must specify a positive number of MaxMachineMapTasks");
}
// and now download the data to a file
var downloadQueueDataBlock = new TransformManyBlock <DownloadMonthlyQueueDataInput, KustoQueueData>(
i =>
{
var action = new DownloadMonthlyQueueData(arguments.AppConfig);
var result = action.PerformAction(i);
// its not worth it to continue if this fails
if (!result.ExecutionStatus)
{
s_logger.Error(result.Exception, "Could not download queue data");
// just throw the exception here
throw result.Exception;
}
return(result.Result.Queues);
}
);
// this is the one that builds machine maps
var createMachineMapBlock = new ActionBlock <KustoQueueData>(
i =>
{
var action = new BuildQueueToMachineMap(arguments.AppConfig);
var result = action.PerformAction(new BuildQueueToMachineMapInput(arguments.Year, arguments.Month, arguments.OutputDirectory, i.QueueName));
// its not worth it to continue if this fails
if (!result.ExecutionStatus)
{
s_logger.Error(result.Exception, $"Could not create machine map for queue {i.QueueName}");
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxMachineMapTasks
}
);
// link
downloadQueueDataBlock.LinkTo(createMachineMapBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
// create the input
var input = new DownloadMonthlyQueueDataInput(arguments.Year, arguments.Month, arguments.OutputDirectory);
// post
downloadQueueDataBlock.Post(input);
// complete
downloadQueueDataBlock.Complete();
// and wait
if (!arguments.IncludeMachineMap)
{
downloadQueueDataBlock.Completion.Wait();
}
else
{
createMachineMapBlock.Completion.Wait();
}
// done
}
public async Task TestOrdering_Async_OrderedDisabled(bool trustedEnumeration)
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded, EnsureOrdered = false };
var tasks = new TaskCompletionSource<IEnumerable<int>>[10];
for (int i = 0; i < tasks.Length; i++)
{
tasks[i] = new TaskCompletionSource<IEnumerable<int>>();
}
var tb = new TransformManyBlock<int, int>(i => tasks[i].Task, options);
tb.PostRange(0, tasks.Length);
for (int i = tasks.Length - 1; i >= 0; i--)
{
tasks[i].SetResult(trustedEnumeration ?
new[] { i } :
Enumerable.Repeat(i, 1));
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
private static void ClassifyInstances(Args arguments)
{
// and a couple of checks here
Contract.Requires(arguments.OutputDirectory != null, "You must specify an output directory");
Contract.Requires(arguments.InputDirectory != null, "You must specify an input directory");
Contract.Requires(Directory.Exists(arguments.OutputDirectory), "The output directory must exist");
Contract.Requires(Directory.Exists(arguments.InputDirectory), "The input directory must exist");
var sharedCount = 0;
var nonSharedCount = 0;
try
{
// load the classifier first
s_logger.Info("Loading classifier...");
// work on the content store
s_logger.Info($"Initializing store at [{arguments.OutputDirectory}]");
var store = new RocksDbContentPlacementPredictionStore(arguments.OutputDirectory, true);
var opContext = new OperationContext(new Context(new LogWrapper(s_logger)));
// init it
var initialized = store.StartupAsync(opContext);
initialized.Wait();
// and check
if (!initialized.Result)
{
s_logger.Error($"Could not initialize RocksDbContentPlacementPredictionStore at [{arguments.OutputDirectory}]");
}
var classifier = new ContentPlacementClassifier(arguments.AppConfig.ClassifierConfiguration);
// create the pipeline. The first step here is to parse the input files, and we can do this in parallel
var buildArtifactParsingBlock = new TransformManyBlock <ParseBuildArtifactsInput, KeyValuePair <string, IReadOnlyList <ArtifactWithBuildMeta> > >(i =>
{
var action = new ParseBuildArtifacts();
var result = action.PerformAction(i);
if (result.ExecutionStatus)
{
return(result.Result.ArtifactsByHash.ToList());
}
else
{
s_logger.Error(result.Exception, $"Error when parsing [{i.BuildArtifactsFile}]");
throw result.Exception;
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxBuildParsingTasks
}
);
// then, when we have one, we linearize it
var linearizeBlock = new TransformBlock <KeyValuePair <string, IReadOnlyList <ArtifactWithBuildMeta> >, TimedActionResult <LinearizeArtifactsOutput> >(i =>
{
var action = new LinearizeArtifacts();
return(action.PerformAction(new LinearizeArtifactsInput(i.Key, i.Value)));
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxArtifactLinearizationTasks
}
);
// and we classify them
var classifyBlock = new ActionBlock <TimedActionResult <LinearizeArtifactsOutput> >(i =>
{
// i have an ml instance here
if (i.ExecutionStatus)
{
var cpInstance = new ContentPlacementInstance()
{
Artifact = i.Result.Linear.AsInstance(), // using the default utility method
QueueName = i.Result.Linear.Queues.First() // the first here is enough, since its always one!
};
var result = classifier.Classify(cpInstance);
if (result.Succeeded)
{
var selectedMachines = result.Value;
foreach (var path in i.Result.Linear.ReportedPaths)
{
store.StoreResult(opContext, path, selectedMachines);
Interlocked.Add(ref sharedCount, 1);
}
}
else
{
Interlocked.Add(ref nonSharedCount, 1);
}
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxArtifactClassificationTasks
}
);
// link them
var numParsingTasks = 0;
buildArtifactParsingBlock.LinkTo(linearizeBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
linearizeBlock.LinkTo(classifyBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
// do now we can post to the initial queue
foreach (var file in Directory.EnumerateFiles(arguments.InputDirectory, "*.json"))
{
buildArtifactParsingBlock.Post(new ParseBuildArtifactsInput(file));
++numParsingTasks;
}
s_logger.Info($"Posted {numParsingTasks} parsing tasks, processing");
// now wait
buildArtifactParsingBlock.Complete();
classifyBlock.Completion.Wait();
// and now we should snapshot
var snapshotDir = Path.Combine(arguments.OutputDirectory, "Snap");
Directory.CreateDirectory(snapshotDir);
s_logger.Info($"Done, snapshoting to [{snapshotDir}]");
var result = store.CreateSnapshot(opContext, snapshotDir);
// done
}
finally
{
var total = 1.0 * (sharedCount + nonSharedCount);
var percentage = (1.0 * sharedCount) / total;
s_logger.Info($"Stats: shared={sharedCount} ({percentage}), nonShared={nonSharedCount}, total={total}");
}
}
private static async Task Main(string[] args)
{
var baseUri = new Uri(args[0]);
var outputSize = args.Length > 1
? int.Parse(args[1])
: DefaultOutputSize;
var downloadString = new TransformBlock <Uri, string>(
async uri => {
using var client = new HttpClient();
return(await client.GetStringAsync(uri));
});
var visitedUris = new HashSet <Uri>();
var findUris = new TransformManyBlock <string, Uri>(
html => AnchorRef.Matches(html)
.Cast <Match>()
.Select(m => new Uri(m.Value, UriKind.RelativeOrAbsolute))
.Select(uri => uri.IsAbsoluteUri ? uri : new Uri(baseUri, uri))
.Where(uri => visitedUris.Add(uri)),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
});
var broadcastUris = new BroadcastBlock <Uri>(uri => uri);
var collectResults = new BatchBlock <Uri>(outputSize);
downloadString.LinkTo(findUris);
findUris.LinkTo(broadcastUris);
broadcastUris.LinkTo(collectResults);
broadcastUris.LinkTo(
downloadString,
uri => baseUri.IsBaseOf(uri) && uri != baseUri);
downloadString.Post(baseUri);
try
{
var uris = await collectResults.ReceiveAsync(TimeSpan.FromSeconds(5));
PrintResults(uris);
}
catch (InvalidOperationException exception) // time out
{
PrintException(exception);
}
finally
{
downloadString.Complete();
}
try
{
await downloadString.Completion;
}
catch (AggregateException exception)
{
PrintException(exception);
}
catch (Exception exception)
{
PrintException(exception);
}
}
/// <inheritdoc/>
public async Task RunAsync(ProcessMode processMode, CancellationToken cancellationToken)
{
if (_messageEncoder == null)
{
throw new NotInitializedException();
}
try {
if (IsRunning)
{
return;
}
IsRunning = true;
_encodingBlock = new TransformManyBlock <DataSetMessageModel[], NetworkMessageModel>(
async input => {
try {
if (_dataSetMessageBufferSize == 1)
{
return(await _messageEncoder.EncodeAsync(input, _maxEncodedMessageSize - _encodedMessageSizeOverhead).ConfigureAwait(false));
}
else
{
return(await _messageEncoder.EncodeBatchAsync(input, _maxEncodedMessageSize - _encodedMessageSizeOverhead).ConfigureAwait(false));
}
}
catch (Exception e) {
_logger.Error(e, "Encoding failure");
return(Enumerable.Empty <NetworkMessageModel>());
}
},
new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken,
});
_batchDataSetMessageBlock = new BatchBlock <DataSetMessageModel>(
_dataSetMessageBufferSize,
new GroupingDataflowBlockOptions {
CancellationToken = cancellationToken,
});
_batchNetworkMessageBlock = new BatchBlock <NetworkMessageModel>(
_networkMessageBufferSize,
new GroupingDataflowBlockOptions {
CancellationToken = cancellationToken,
});
_sinkBlock = new ActionBlock <NetworkMessageModel[]>(
async input => {
if (input != null && input.Any())
{
await _messageSink.SendAsync(input).ConfigureAwait(false);
}
else
{
_logger.Information("Sink block in engine {Name} triggered with empty input",
Name);
}
},
new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken,
});
_batchDataSetMessageBlock.LinkTo(_encodingBlock);
_encodingBlock.LinkTo(_batchNetworkMessageBlock);
_batchNetworkMessageBlock.LinkTo(_sinkBlock);
_messageTrigger.OnMessage += MessageTriggerMessageReceived;
if (_diagnosticInterval > TimeSpan.Zero)
{
_diagnosticsOutputTimer.Change(_diagnosticInterval, _diagnosticInterval);
}
await _messageTrigger.RunAsync(cancellationToken).ConfigureAwait(false);
}
finally {
IsRunning = false;
_messageTrigger.OnMessage -= MessageTriggerMessageReceived;
_diagnosticsOutputTimer.Change(Timeout.Infinite, Timeout.Infinite);
_batchTriggerIntervalTimer.Change(Timeout.Infinite, Timeout.Infinite);
await _sinkBlock.Completion;
}
}
private static ITargetBlock <string> CreateTextProcessingPipeline(string inputPath, out Task completionTask)
{
int fileCount = Directory.GetFiles(inputPath, "*.txt").Length;
var getFilenames = new TransformManyBlock <string, string>(
path =>
{
return(Directory.GetFiles(path, "*.txt"));
});
var getFileContents = new TransformBlock <string, string>(async(filename) =>
{
Console.WriteLine("Begin: getFileContents");
using var streamReader = new StreamReader(filename);
return(await streamReader.ReadToEndAsync());
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
var analyzeContents = new TransformBlock <string, Dictionary <string, ulong> >(contents =>
{
Console.WriteLine("Begin: analyzeContents;");
var frequencies = new Dictionary <string, ulong>(10000, StringComparer.OrdinalIgnoreCase);
var matches = WordRegex.Matches(contents);
foreach (Match match in matches)
{
if (!frequencies.TryGetValue(match.Value, out ulong currentValue))
{
currentValue = 0;
}
frequencies[match.Value] = currentValue + 1;
}
Console.WriteLine("End: analyzeContents");
return(frequencies);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
});
var eliminateIgnoredWords = new TransformBlock <Dictionary <string, ulong>, Dictionary <string, ulong> >(input =>
{
foreach (var word in IgnoreWords)
{
input.Remove(word);
}
return(input);
});
var batch = new BatchBlock <Dictionary <string, ulong> >(fileCount);
// 所有的处理都集中在这一个点上
var combineFrequencies = new TransformBlock <Dictionary <string, ulong>[], List <KeyValuePair <string, ulong> > >(inputs =>
{
Console.WriteLine("Begin: combiningFrequencies");
var sortedList = new List <KeyValuePair <string, ulong> >();
var combineFrequencies = new Dictionary <string, ulong>(10000, StringComparer.OrdinalIgnoreCase);
foreach (var input in inputs)
{
foreach (var kvp in input)
{
if (!combineFrequencies.TryGetValue(kvp.Key, out ulong currentFrequency))
{
currentFrequency = 0;
}
combineFrequencies[kvp.Key] = currentFrequency + kvp.Value;
}
}
foreach (var kvp in combineFrequencies)
{
sortedList.Add(kvp);
}
sortedList.Sort((a, b) =>
{
return(-a.Value.CompareTo(b.Value));
});
Console.WriteLine("End: combineFrequencies");
return(sortedList);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
var printTopTen = new ActionBlock <List <KeyValuePair <string, ulong> > >(input =>
{
for (int i = 0; i < input.Count; i++)
{
Console.WriteLine($"{input[i].Key} - {input[i].Value}");
}
getFilenames.Complete();
});
// 链接块
getFilenames.LinkTo(getFileContents);
getFileContents.LinkTo(analyzeContents);
analyzeContents.LinkTo(eliminateIgnoredWords);
eliminateIgnoredWords.LinkTo(batch);
batch.LinkTo(combineFrequencies);
combineFrequencies.LinkTo(printTopTen);
completionTask = getFilenames.Completion;
return(getFilenames);
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory =
options =>
{
TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
return(Enumerable.Range(0, 10));
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(new[] { i });
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions {
CancellationToken = cts.Token
};
var t = new TransformManyBlock <int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList <int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock <int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (IList <int> list in new IList <int>[]
{
new int[1],
new List <int>()
{
0
},
new Collection <int>()
{
0
}
})
{
int nextExpectedValue = 1;
TransformManyBlock <int, int> tmb1 = null;
tmb1 = new TransformManyBlock <int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return((IEnumerable <int>)null);
}
else if (dop == 1)
{
list[0] = i + 1;
return((IEnumerable <int>)list);
}
else if (list is int[])
{
return(new int[1] {
i + 1
});
}
else if (list is List <int> )
{
return(new List <int>()
{
i + 1
});
}
else
{
return new Collection <int>()
{
i + 1
}
};
}, dbo);
TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return(i);
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock <int, int>(i =>
{
if (i < ITERS - 1)
{
throw new OperationCanceledException();
}
if (mode == 0)
{
return new int[] { i }
}
;
else if (mode == 1)
{
return new List <int>()
{
i
}
}
;
else
{
return(Enumerable.Repeat(i, 1));
}
}, dbo);
var ab = new ActionBlock <int>(i =>
{
if (i != ITERS - 1)
{
localPassed = false;
}
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++)
{
tmb.SendAsync(i).Wait();
}
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory =
options =>
{
TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(new[] { i });
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
return(null);
}
return(Task.Run(() => (IEnumerable <int>) new[] { i }));
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return(null);
}
return(Task.Run(() => (IEnumerable <int>) new[] { i }));
}), new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2
});
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => Task.Run <IEnumerable <int> >(new Func <IEnumerable <int> >(() => { throw new InvalidOperationException(); }))));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (IList <int> list in new IList <int>[]
{
new int[1],
new List <int>()
{
0
},
new Collection <int>()
{
0
}
})
{
int nextExpectedValue = 1;
TransformManyBlock <int, int> tmb1 = null;
tmb1 = new TransformManyBlock <int, int>(i =>
{
return(Task.Run(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable <int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable <int>)list;
}
else if (list is int[])
{
return new int[1] {
i + 1
};
}
else if (list is List <int> )
{
return new List <int>()
{
i + 1
};
}
else
{
return new Collection <int>()
{
i + 1
}
};
}));
}, dbo);
TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return(i);
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock <int, int>(i =>
{
var cts = new CancellationTokenSource();
return(Task.Run(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0)
{
return new int[] { i }
}
;
else if (mode == 1)
{
return new List <int>()
{
i
}
}
;
else
{
return Enumerable.Repeat(i, 1);
}
}, cts.Token));
}, dbo);
var ab = new ActionBlock <int>(i =>
{
if (i != ITERS - 1)
{
localPassed = false;
}
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++)
{
tmb.SendAsync(i).Wait();
}
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
/// <summary>
/// Creates a dataflow block from current configuration.
/// </summary>
protected override IPropagatorBlock <TInput, TOutput> CreateBlock()
{
TransformManyBlock <TInput, TOutput> block;
if (this.processAsync != null)
{
block = new TransformManyBlock <TInput, TOutput>
(async input =>
{
// ReSharper disable once CompareNonConstrainedGenericWithNull
if (input == null)
{
return(new TOutput[0]);
}
try
{
var result = await this.processAsync(input).ConfigureAwait(false);
return(result);
}
catch (Exception ex)
{
var logger = input as IDataflowErrorLogger;
if (logger != null)
{
logger.OnException(ex);
}
else
{
this.DefaultExceptionLogger?.Invoke(ex, input);
}
return(new TOutput[0]);
}
},
this.options);
}
else
{
block = new TransformManyBlock <TInput, TOutput>
(input =>
{
// ReSharper disable once CompareNonConstrainedGenericWithNull
if (input == null)
{
return(new TOutput[0]);
}
try
{
var result = this.processSync(input);
return(result);
}
catch (Exception ex)
{
var logger = input as IDataflowErrorLogger;
if (logger != null)
{
logger.OnException(ex);
}
else
{
this.DefaultExceptionLogger?.Invoke(ex, input);
}
return(new TOutput[0]);
}
},
this.options);
}
return(block);
}
public async Task HandleAsync(CommandContext context, NextDelegate next)
{
if (context.Command is BulkUpdateContents bulkUpdates)
{
if (bulkUpdates.Jobs?.Length > 0)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2)
};
var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task =>
{
try
{
return(await CreateCommandsAsync(task));
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command =>
{
try
{
await ExecuteCommandAsync(command);
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
createCommandsBlock.BidirectionalLinkTo(executeCommandBlock);
contextProvider.Context.Change(b => b
.WithoutContentEnrichment()
.WithoutCleanup()
.WithUnpublished(true)
.WithoutTotal());
var requestedSchema = bulkUpdates.SchemaId.Name;
var results = new ConcurrentBag <BulkUpdateResultItem>();
for (var i = 0; i < bulkUpdates.Jobs.Length; i++)
{
var task = new BulkTask(
context.CommandBus,
requestedSchema,
i,
bulkUpdates.Jobs[i],
bulkUpdates,
results);
if (!await createCommandsBlock.SendAsync(task))
{
break;
}
}
createCommandsBlock.Complete();
await executeCommandBlock.Completion;
context.Complete(new BulkUpdateResult(results));
}
else
{
context.Complete(new BulkUpdateResult());
}
}
else
{
await next(context);
}
}
public void Dispose()
{
if (this.evalBlock != null)
{
this.evalBlock.Complete();
this.evalBlock.Completion.Wait(TimeSpan.FromMinutes(1));
this.evalBlock = null;
}
if (this.evalBlockDisposable != null)
{
this.evalBlockDisposable.Dispose();
this.evalBlockDisposable = null;
}
}
/// <summary>
/// Purges all blobs set to expire prior to the specified date.
/// </summary>
/// <param name="deleteBlobsExpiringBefore">
/// All blobs scheduled to expire prior to this date will be purged. The default value
/// is interpreted as <see cref="DateTime.UtcNow"/>.
/// </param>
/// <returns>The task representing the asynchronous operation.</returns>
public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime))
{
if (deleteBlobsExpiringBefore == default(DateTime))
{
deleteBlobsExpiringBefore = DateTime.UtcNow;
}
Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required.");
var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>(
async c =>
{
var results = await c.ListBlobsSegmentedAsync();
return from directory in results.OfType<CloudBlobDirectory>()
let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/'))
where expires < deleteBlobsExpiringBefore
select directory;
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>(
async directory =>
{
var results = await directory.ListBlobsSegmentedAsync();
return results.OfType<CloudBlockBlob>();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<CloudBlockBlob>(
blob => blob.DeleteAsync(),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true });
deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredDirectoriesBlock.Post(this.container);
searchExpiredDirectoriesBlock.Complete();
await deleteBlobBlock.Completion;
}
public void AsyncCancelledTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformManyBlock<int, int> (
i =>
{
var tcs = new TaskCompletionSource<IEnumerable<int>> ();
tcs.SetCanceled ();
return tcs.Task;
}, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
}
