public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Signaling completion to the source block.");
bufferBlock.Complete();
Console.WriteLine("Done.");
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(200);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(500);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
Console.WriteLine("Waiting for the completion to be propagated through the network...");
actionBlock.Completion.ContinueWith(t =>
{
Console.WriteLine("Finished processing.");
Console.WriteLine(string.Format("Completion status: {0}.", t.Status));
}).Wait();
}
public BatchedInsert()
{
Output = new TransformBlock<object, List<object>>(obj =>
{
if (objects.Count > BatchSize)
objects.Clear();
if(objects.Count < BatchSize)
{
objects.Add(obj);
return null;
}
return objects;
});
DatabaseInserter = new ActionBlock<object>(objs =>
{
if (objs == null)
return;
var toInsert = (List<object>)objs;
toInsert = new List<object>(toInsert);
Mapping.Insert(toInsert);
});
(Output as TransformBlock<object, List<object>>).LinkTo(DatabaseInserter);
}
public static async Task ValidateBlockAsync(ICoreStorage coreStorage, ICoreRules rules, Chain newChain, ISourceBlock<ValidatableTx> validatableTxes, CancellationToken cancelToken = default(CancellationToken))
{
// tally transactions
object finalTally = null;
var txTallier = new TransformBlock<ValidatableTx, ValidatableTx>(
validatableTx =>
{
var runningTally = finalTally;
rules.TallyTransaction(newChain, validatableTx, ref runningTally);
finalTally = runningTally;
return validatableTx;
});
validatableTxes.LinkTo(txTallier, new DataflowLinkOptions { PropagateCompletion = true });
// validate transactions
var txValidator = InitTxValidator(rules, newChain, cancelToken);
// begin feeding the tx validator
txTallier.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true });
// validate scripts
var scriptValidator = InitScriptValidator(rules, newChain, cancelToken);
// begin feeding the script validator
txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true });
//TODO
await PipelineCompletion.Create(
new Task[] { },
new IDataflowBlock[] { validatableTxes, txTallier, txValidator, scriptValidator });
// validate overall block
rules.PostValidateBlock(newChain, finalTally);
}
public void Run()
{
var cts = new CancellationTokenSource();
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>(new DataflowBlockOptions { CancellationToken = cts.Token });
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Scheduling cancellation after 5 seconds.");
cts.CancelAfter(TimeSpan.FromSeconds(5));
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1, CancellationToken = cts.Token });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10, CancellationToken = cts.Token });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
var t1 = bufferBlock.Completion.ContinueWith(t => Console.WriteLine("Buffer block status: {0}", t.Status));
var t2 = actionBlock.Completion.ContinueWith(t => Console.WriteLine("Action block status: {0}", t.Status));
Console.WriteLine("Waiting for the network to finish.");
Task.WaitAll(t1, t2);
}
public static ITargetBlock<string> SetupPipeline()
{
var fileNamesForPath = new TransformBlock<string, IEnumerable<string>>(
path =>
{
return GetFileNames(path);
});
var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
fileNames =>
{
return LoadLines(fileNames);
});
var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
lines2 =>
{
return GetWords(lines2);
});
var display = new ActionBlock<IEnumerable<string>>(
coll =>
{
foreach (var s in coll)
{
WriteLine(s);
}
});
fileNamesForPath.LinkTo(lines);
lines.LinkTo(words);
words.LinkTo(display);
return fileNamesForPath;
}
public MessagePipeline()
{
linkOptions = new DataflowLinkOptions { PropagateCompletion = true };
buildMessage = new TransformBlock<object, Messaging.Message>(
x => {
Console.WriteLine("buildMessage| message: {0}", x);
return new Messaging.Message { Body = x };
});
logMessage = new TransformBlock<Messaging.Message, Messaging.Message>
(x => {
Console.WriteLine("logMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body);
return x;
});
sendMessage = new TransformBlock<Messaging.Message, Messaging.Message>(
x => {
Console.WriteLine("sendMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body);
return x;
});
buildMessage.LinkTo(logMessage, linkOptions);
logMessage.LinkTo(sendMessage, linkOptions);
}
public void Start()
{
var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink);
var source = new BufferBlock<GetPageMessage>();
var linkOptions = new DataflowLinkOptions {PropagateCompletion = false};
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var worker = new TransformBlock<GetPageMessage, PageResultMessage>(
(Func<GetPageMessage, PageResultMessage>)Worker, options);
source.LinkTo(worker, linkOptions);
worker.LinkTo(sink, linkOptions);
}
foreach (var url in UrlList.Urls)
{
source.Post(new GetPageMessage{ Url = url });
}
source.Complete();
sink.Completion.Wait();
}
public void TestTransformBlockConstructor()
{
// SYNC
{
// without option
var block = new TransformBlock<int, string>(i => i.ToString());
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with not cancelled token and default scheduler
block = new TransformBlock<int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new TransformBlock<int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
}
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// without option
var block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString()));
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with not cancelled token and default scheduler
block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
}
}
public MatchPipeline(RiotApi riotApi, RiotQuerySettings querySettings, IMatchDetailProcessor matchDetailProcessor)
{
api = riotApi;
this.querySettings = querySettings;
this.matchDetailProcessor = matchDetailProcessor;
queryQueues.Add(querySettings.Queue);
// Create match producer
if (!querySettings.NoDownload)
{
playerMatchProducer = new PlayerMatchProducer(api, querySettings, queryQueues, testSynchronizer);
// Create blocks
ConsumeMatchBlock = new TransformBlock<MatchSummary, MatchDetail>(
async match => await ConsumeMatch(match),
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 5 });
}
ConsumeMatchDetailBlock = new ActionBlock<MatchDetail>(
async match => await matchDetailProcessor.ConsumeMatchDetail(match),
new ExecutionDataflowBlockOptions() {
MaxDegreeOfParallelism = matchDetailProcessor.MaxDegreeOfParallelism
});
// Link blocks
if (!querySettings.NoDownload)
{
playerMatchProducer.MatchProducerBlock.LinkTo(ConsumeMatchBlock, new DataflowLinkOptions() { PropagateCompletion = false });
ConsumeMatchBlock.LinkTo(ConsumeMatchDetailBlock, new DataflowLinkOptions() { PropagateCompletion = false }, match => match != null);
ConsumeMatchBlock.LinkTo(DataflowBlock.NullTarget<MatchDetail>(), new DataflowLinkOptions() { PropagateCompletion = false });
}
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
_waitHandle.Signal();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock);
transformBlock.LinkTo(actionBlock);
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
_waitHandle.Wait();
}
/// <summary>
/// Search Asynchrony many extension in all of Fixed and Removable Disks.
/// </summary>
/// <param name="targetExtensions">Some file extensions for use search pattern.</param>
/// <example>
/// FileExtension example:
/// {".jpg", 646546Byte, 646Byte}
/// {".pdf", 25464645546Byte, 60000Byte}
/// </example>
/// <returns>A sorted list of detected files</returns>
public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS)
{
return await Task.Run(() =>
{
searchComplete = false;
//
Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---"));
List<FileInfo> _result = new List<FileInfo>();
//
// Find specific folders from windows drives instead of the total drive.
//
FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation();
//
// Set Data-flow
//
TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir =>
{
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchBeginning,
"Searching {0} ...", dir.FullName));
List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS);
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchCompleted,
"The Search {0} was completed!", dir.FullName));
return res;
}, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount });
ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst));
//
// Search specific folders from windows drives instead of the total drive.
//
try
{
TB.LinkTo(AB);
ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount };
var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir));
TB.Complete();
TB.Completion.Wait();
}
catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); }
searchComplete = true;
Reporter("DiskProbing", new ReportEventArgs("DiskProbing",
ReportCodes.DiskProbingFinished,
"---{Search Disks Finished}---"));
LastSearchResult = _result;
return _result;
});
}
public void Initialise()
{
_systemMetrics = new Mock<ISystemMetricsService>();
_log = new Mock<ILog>();
_block = MessageParserBlockFactory.CreateMessageParserBlock(new CancellationToken(),
_systemMetrics.Object,
_log.Object);
}
public static ISourceBlock<ValidatableTx> ReplayRollbackUtxo(ICoreStorage coreStorage, IChainState chainState, ChainedHeader replayBlock, CancellationToken cancelToken = default(CancellationToken))
{
// replaying rollback of an on-chain block, use the chainstate tx index for replay, same as replaying forward
if (chainState.Chain.BlocksByHash.ContainsKey(replayBlock.Hash))
{
return ReplayFromTxIndex(coreStorage, chainState, replayBlock, replayForward: false, cancelToken: cancelToken);
}
// replaying rollback of an off-chain (re-org) block, use the unminted information for replay
else
{
IImmutableList<UnmintedTx> unmintedTxesList;
if (!chainState.TryGetBlockUnmintedTxes(replayBlock.Hash, out unmintedTxesList))
{
//TODO if a wallet/monitor were to see a chainstate block that wasn't flushed to disk yet,
//TODO and if bitsharp crashed, and if the block was orphaned: then the orphaned block would
//TODO not be present in the chainstate, and it would not get rolled back to generate unminted information.
//TODO DeferredChainStateCursor should be used in order to re-org the chainstate in memory and calculate the unminted information
throw new MissingDataException(replayBlock.Hash);
}
var unmintedTxes = ImmutableDictionary.CreateRange(
unmintedTxesList.Select(x => new KeyValuePair<UInt256, UnmintedTx>(x.TxHash, x)));
var lookupLoadingTx = new TransformBlock<DecodedBlockTx, ValidatableTx>(
blockTx =>
{
var tx = blockTx.Transaction;
var txIndex = blockTx.Index;
var prevTxOutputs = ImmutableArray.CreateBuilder<PrevTxOutput>(!blockTx.IsCoinbase ? tx.Inputs.Length : 0);
if (!blockTx.IsCoinbase)
{
UnmintedTx unmintedTx;
if (!unmintedTxes.TryGetValue(tx.Hash, out unmintedTx))
throw new MissingDataException(replayBlock.Hash);
prevTxOutputs.AddRange(unmintedTx.PrevTxOutputs);
}
return new ValidatableTx(blockTx, replayBlock, prevTxOutputs.MoveToImmutable());
});
IEnumerator<BlockTx> blockTxes;
if (!coreStorage.TryReadBlockTransactions(replayBlock.Hash, out blockTxes))
{
throw new MissingDataException(replayBlock.Hash);
}
var blockTxesBuffer = new BufferBlock<DecodedBlockTx>();
blockTxesBuffer.LinkTo(lookupLoadingTx, new DataflowLinkOptions { PropagateCompletion = true });
blockTxesBuffer.SendAndCompleteAsync(blockTxes.UsingAsEnumerable().Select(x => x.Decode()).Reverse(), cancelToken).Forget();
return lookupLoadingTx;
}
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
static void Main(string[] args)
{
string s =
"http://cn.bing.com/search?q=MD5CryptoServiceProvider+slow&qs=n&pq=md5cryptoserviceprovider+slow&sc=0-25&sp=-1&sk=&cvid=67d40cbd8c424d55a3db83e6e9868267&first=51&FORM=PERE4";
using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider())
{
byte[] inBytes = Encoding.UTF8.GetBytes(s);
var bytes = md5.ComputeHash(inBytes);
Console.WriteLine(bytes.Length);
}
var splitter = new TransformBlock<string, KeyValuePair<string, int>>(
input =>
{
var splitted = input.Split('=');
return new KeyValuePair<string, int>(splitted[0], int.Parse(splitted[1]));
});
var dict = new Dictionary<string, int>();
var aggregater = new ActionBlock<KeyValuePair<string, int>>(
pair =>
{
int oldValue;
dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value;
});
splitter.LinkTo(aggregater, new DataflowLinkOptions() { PropagateCompletion = true});
splitter.Post("a=1");
splitter.Post("b=2");
splitter.Post("a=5");
splitter.Complete();
aggregater.Completion.Wait();
Console.WriteLine("sum(a) = {0}", dict["a"]); //prints sum(a) = 6
//CalcAsync().Wait();
//SlowFlowAsync().Wait();
//FailDemoAsync().Wait();
//TransformAndLinkDemo().Wait();
//LinkLeftToDemo().Wait();
//CircularFlowAutoComplete().Wait();
//RecorderDemo().Wait();
BulkInserterDemo().Wait();
//BulkInserterDemo2().Wait();
//BroadcasterDemo().Wait();
//MyLoggerDemo().Wait();
//ETLLookupDemo().Wait();
}
private static TransformBlock<int, string> ConstructTransformWithNMessages(int messagesCount)
{
var block = new TransformBlock<int, string>(i => i.ToString());
for (int i = 0; i < messagesCount; i++)
{
block.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return block;
}
/// <summary>
/// Initials the transmitter asynchronous.
/// Check the server and then database existence and ...
/// </summary>
public static async Task InitialTransmitterAsync()
{
await ServerValidatorAsync();
ErrorListenerTransformBlock = new TransformBlock<ProxyError, Tuple<ProxyError, bool>>(
async (e) => await TransmitOneError(e),
new ExecutionDataflowBlockOptions()
{
MaxMessagesPerTask = 1,
MaxDegreeOfParallelism = 1
});
ErrorListenerTransformBlock.LinkTo(CacheController.AcknowledgeActionBlock);
}
public async Task TransformThroughFilterToAction()
{
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }, i => true);
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.Equal(expected: Iterations, actual: completedCount);
}
public void DeferredUsageTest ()
{
int[] array = new int[10];
var action = new ActionBlock<int> (i => array[Math.Abs (i)] = i);
var block = new TransformBlock<int, int> (i => -i);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
Thread.Sleep (300);
block.LinkTo (action);
Thread.Sleep (100);
CollectionAssert.AreEqual (new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public void BasicUsageTest ()
{
int[] array = new int[10];
var evt = new ManualResetEventSlim (false);
ActionBlock<int> action = new ActionBlock<int> ((i) => { array[Math.Abs (i)] = i; evt.Set (); });
TransformBlock<int, int> block = new TransformBlock<int, int> (i => -i);
block.LinkTo (action);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
evt.Wait ();
CollectionAssert.AreEqual (new int[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public Statsd(string serviceName = null)
{
_log.Info("statsd.net starting.");
_tokenSource = new CancellationTokenSource();
_shutdownComplete = new ManualResetEvent(false);
SuperCheapIOC.Add(_log);
var systemInfoService = new SystemInfoService();
SuperCheapIOC.Add(systemInfoService as ISystemInfoService);
serviceName = serviceName ?? systemInfoService.HostName;
var systemMetricsService = new SystemMetricsService("statsdnet", serviceName);
SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService);
/**
* The flow is:
* Listeners ->
* Message Parser ->
* router ->
* Aggregator ->
* Broadcaster ->
* Backends
*/
// Initialise the core blocks
_router = new StatsdMessageRouterBlock();
_messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token,
SuperCheapIOC.Resolve<ISystemMetricsService>(),
_log);
_messageParser.LinkTo(_router);
_messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () =>
{
_log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster.");
_messageBroadcaster.Complete();
});
_messageBroadcaster = new BroadcastBlock<Bucket>(Bucket.Clone);
_messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () =>
{
_log.Info("MessageBroadcaster: Completion signaled. Notifying all backends.");
_backends.ForEach(q => q.Complete());
});
// Add the broadcaster to the IOC container
SuperCheapIOC.Add<BroadcastBlock<Bucket>>(_messageBroadcaster);
systemMetricsService.SetTarget(_messageBroadcaster);
_backends = new List<IBackend>();
_listeners = new List<IListener>();
}
static ITargetBlock<string> SetupPipeline()
{
var fileNames = new TransformBlock<string, IEnumerable<string>>(path =>
{
try
{
return GetFileNames(path);
}
catch (OperationCanceledException)
{
return Enumerable.Empty<string>();
}
});
var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(input =>
{
try
{
return LoadLines(input);
}
catch (OperationCanceledException)
{
return Enumerable.Empty<string>();
}
});
var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(lines2 =>
{
return GetWords(lines2);
});
var display = new ActionBlock<IEnumerable<string>>(coll =>
{
foreach (var s in coll)
{
Console.WriteLine(s);
}
});
fileNames.LinkTo(lines);
lines.LinkTo(words);
words.LinkTo(display);
// fileNames.LinkTo(loadLines, fn => fn.Count() > 0);
return fileNames;
}
public void ProcessFile(string fileName)
{
var inputBlock = new BufferBlock<string>();
var readBlock = new TransformBlock<string, string>(
(input) =>
{
Console.WriteLine("Loading " + input);
return File.ReadAllText(input);
});
var compileBlock = new TransformBlock<string, string>(
(input) =>
{
Console.WriteLine("Processing...");
var client = new WebClient();
client.Headers.Add("content-type", "application/x-www-form-urlencoded");
string apiData = string.Format(Data, HttpUtility.UrlEncode(input));
return client.UploadString(Url, apiData);
});
var convertBlock = new TransformBlock<string, XDocument>(
(input) =>
{
Console.WriteLine("Converting to XDocument...");
var xml = XDocument.Parse(input);
return xml;
});
var outputBlock = new ActionBlock<XDocument>(
(input) =>
{
Console.WriteLine("Writing compressed-" + fileName);
XElement compiledCode = input.Element("compilationResult").Element("compiledCode");
File.WriteAllText("compressed-" + fileName, compiledCode.Value);
});
inputBlock.LinkTo(readBlock, new DataflowLinkOptions { PropagateCompletion = true });
readBlock.LinkTo(compileBlock, new DataflowLinkOptions { PropagateCompletion = true });
compileBlock.LinkTo(convertBlock, new DataflowLinkOptions { PropagateCompletion = true });
convertBlock.LinkTo(outputBlock, new DataflowLinkOptions { PropagateCompletion = true });
inputBlock.Post(fileName);
inputBlock.Complete();
outputBlock.Completion.Wait();
}
public void AsyncNullTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformBlock<int, int> (
i => null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
public CompletionDemo1()
{
broadCaster = new BroadcastBlock<int>(
i =>
{
return i;
}).ToDataflow();
transformBlock1 = new TransformBlock<int, string>(
i =>
{
Console.WriteLine("1 input count: " + transformBlock1.InputCount);
Thread.Sleep(50);
return ("1_" + i);
});
transformBlock2 = new TransformBlock<int, string>(
i =>
{
Console.WriteLine("2 input count: " + transformBlock2.InputCount);
Thread.Sleep(20);
return ("2_" + i);
});
processor = new ActionBlock<string>(
i =>
{
Console.WriteLine(i);
}).ToDataflow();
/** rather than TPL linking
broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true });
broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
**/
//Use DataflowEx linking
var transform1 = transformBlock1.ToDataflow();
var transform2 = transformBlock2.ToDataflow();
broadCaster.LinkTo(transform1);
broadCaster.LinkTo(transform2);
transform1.LinkTo(processor);
transform2.LinkTo(processor);
}
internal static bool TransformThroughFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => true);
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock()
{
var directoryBrowserBlock = new TransformManyBlock<string, string>(path =>
{
var dir = new DirectoryInfo(path);
return dir.EnumerateFileSystemInfos()
.Select(fi => fi.FullName);
});
var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName =>
{
var fi = new FileInfo(fileName);
return new KeyValuePair<string, long>(fileName, fi.Length);
});
directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists);
var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator);
return customBlock;
}
public static void Example1()
{
var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 };
ActionBlock<int> a = new ActionBlock<int>(i =>
{
Thread.Sleep(500);
Console.WriteLine(i);
}, conf);
TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3);
t.LinkTo(a);
for (int i = 0; i < 12; i++)
{
t.Post(i);
}
}
public void Run()
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var tb = new TransformBlock<int, int>(i => i * 2);
var ab = new ActionBlock<int>(i => this.Compute(i), options);
tb.LinkTo(ab);
for (var i = 0; i < 10; i++)
{
tb.Post(i);
}
tb.Complete();
tb.Completion.Wait();
Thread.Sleep(500);
}
public WebSocketListener(IPEndPoint endpoint, WebSocketListenerOptions options)
{
if (options == null)
throw new ArgumentNullException("options");
if (endpoint == null)
throw new ArgumentNullException("endpoint");
_options = options.Clone();
_cancel = new CancellationTokenSource();
_listener = new TcpListener(endpoint);
ConnectionExtensions = new WebSocketConnectionExtensionCollection(this);
Standards = new WebSocketFactoryCollection(this);
Func<Socket, Task<WebSocketNegotiationResult>> negotiate = NegotiateWebSocket;
_negotiationQueue = new TransformBlock<Socket, WebSocketNegotiationResult>(negotiate, new ExecutionDataflowBlockOptions() { CancellationToken = _cancel.Token, MaxDegreeOfParallelism = options.ParallelNegotiations, BoundedCapacity = options.NegotiationQueueCapacity });
_handShaker = new WebSocketHandshaker(Standards, _options);
}
/// <summary>
/// Disposes internal members.
/// </summary>
/// <param name="disposing">Whether the object is being disposed.</param>
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (this.eventSource != null)
{
// Flush the data buffer to upload all remaining events.
this.eventSource.Complete();
this.eventProcessor.Completion.Wait();
this.eventSource = null;
}
if (this.eventUnsubscriber != null)
{
this.eventUnsubscriber.Dispose();
this.eventUnsubscriber = null;
}
this.cancellationTokenSource.Cancel();
}
}
public async Task TestCancellationExceptionsIgnored()
{
var t = new TransformBlock <int, int>(i => {
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(i);
});
t.PostRange(0, 2);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0)
{
Assert.Equal(expected: i, actual: await t.ReceiveAsync());
}
}
await t.Completion;
}
public async Task TestCtorOverloading()
{
Func <object, Task <object> > f = x => Task.FromResult <object>(x);
for (int test = 0; test < 2; test++)
{
TransformBlock <object, object> tf = test == 0 ?
new TransformBlock <object, object>(f) :
new TransformBlock <object, object>((Func <object, object>)f);
var tcs = new TaskCompletionSource <bool>();
ActionBlock <object> a = new ActionBlock <object>(x => {
Assert.Equal(expected: test == 1, actual: x is Task <object>);
tcs.SetResult(true);
});
tf.LinkTo(a);
tf.Post(new object());
await tcs.Task;
}
}
public CalculationDataFlow(ILogger <CalculationDataFlow> logger, IResultProducer resultProducer)
{
_logger = logger;
_resultProducer = resultProducer;
_deserializeBlock = new TransformBlock <byte[], CalculationPack>(msg => MessagePackSerializer.Deserialize <CalculationPack>(msg));
_calculationBlock = new TransformBlock <CalculationPack, CalculationPack>(pack =>
{
Parallel.ForEach(pack.Data, unit =>
{
unit.Equation.Result = CalculationHelper.Calculate(unit.Equation.EqMethod, unit.Equation.Values);
});
return(pack);
});
_publishBlock = new ActionBlock <CalculationPack>(pack =>
{
//TODO publish in kafka topic
//_logger.LogInformation(pack.Id.ToString());
foreach (var eq in pack.Data)
{
_logger.LogInformation($"PackId: {eq.CalcPackId} EqId: {eq.Equation.Id} Result:{eq.Equation.Result}");
}
_resultProducer.ProduceResultAsync(pack);
});
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
_deserializeBlock.LinkTo(_calculationBlock, linkOptions);
_calculationBlock.LinkTo(_publishBlock, linkOptions);
}
public void Process_pipeline_one_input_one_output()
{
var result = string.Empty;
var head = new TransformBlock <string, string>(str => str + "A");
var block2 = new TransformBlock <string, string>(str => str + "B");
var block3 = new TransformBlock <string, string>(str => str + "C");
var block4 = new ActionBlock <string>(str => result = str);
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
head.LinkTo(block2, linkOptions);
block2.LinkTo(block3, linkOptions);
block3.LinkTo(block4, linkOptions);
head.Post(string.Empty);
head.Complete();
block4.Completion.Wait();
result.Should().Be("ABC");
}
static void Main(string[] args)
{
// TransformMany();
// Batching();
// Errors();
// DangerOfCondiionalLinking();
// SingleActionBlockLoadBalance();
var greedy = new ExecutionDataflowBlockOptions();
var nonGreedy = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1
};
ExecutionDataflowBlockOptions options = nonGreedy;
var firstBlock = new ActionBlock <int>(i => Do(i, 1, 2), options);
var secondBlock = new ActionBlock <int>(i => Do(i, 2, 1), options);
var thirdBlock = new ActionBlock <int>(i => Do(i, 3, 2), options);
var transform = new TransformBlock <int, int>(i => i * 2);
transform.LinkTo(firstBlock);
transform.LinkTo(secondBlock);
transform.LinkTo(thirdBlock);
for (int i = 0; i <= 10; i++)
{
transform.Post(i);
}
Console.ReadLine();
}
public MultiThreadRunner(
bool verbose,
List <DiagnosticAnalyzer> analyzers,
Func <ImmutableArray <Diagnostic>, ParsedOptions, ConcurrentDictionary <string, DiagnosticDescriptor>, SarifV2ErrorLogger, int> logDiagnostics,
ParsedOptions parsedOptions,
ConcurrentDictionary <string, DiagnosticDescriptor> descriptors,
SarifV2ErrorLogger logger,
int threads)
: base(analyzers, logDiagnostics, parsedOptions, descriptors, logger)
{
_scanBlock = new TransformBlock <Project, ImmutableArray <Diagnostic> >(async project =>
{
if (verbose)
{
Console.WriteLine($"Starting: {project.FilePath}");
}
return(await GetDiagnostics(project).ConfigureAwait(false));
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = threads,
EnsureOrdered = false,
BoundedCapacity = 32
});
_resultsBlock = new ActionBlock <ImmutableArray <Diagnostic> >(diagnostics =>
{
_count += logDiagnostics(diagnostics, _parsedOptions, descriptors, logger);
},
new ExecutionDataflowBlockOptions
{
EnsureOrdered = false
});
_scanBlock.LinkTo(_resultsBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
}
public void TestTransformBlockConstructor()
{
// SYNC
{
// without option
var block = new TransformBlock <int, string>(i => i.ToString());
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with not cancelled token and default scheduler
block = new TransformBlock <int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions {
MaxMessagesPerTask = 1
});
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new TransformBlock <int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions {
MaxMessagesPerTask = 1, CancellationToken = token
});
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
}
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// without option
var block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString()));
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with not cancelled token and default scheduler
block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString()), new ExecutionDataflowBlockOptions {
MaxMessagesPerTask = 1
});
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString()), new ExecutionDataflowBlockOptions {
MaxMessagesPerTask = 1, CancellationToken = token
});
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock.");
}
}
private static void SynchronousHappySites()
{
string[] urls = new string[]
{
"http://www.bbc.co.uk",
"http://www.cia.gov",
"http://www.theregister.co.uk"
};
var happySites = new List <string>();
var isHappySiteBlock = new TransformBlock <string, Tuple <string, bool> >((Func <string, Tuple <string, bool> >)IsHappy,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
var addToHappySitesBlock = new ActionBlock <Tuple <string, bool> >(
(Action <Tuple <string, bool> >)(tuple => happySites.Add(tuple.Item1)));
isHappySiteBlock.LinkTo(addToHappySitesBlock,
new DataflowLinkOptions()
{
PropagateCompletion = true
},
tuple => tuple.Item2);
isHappySiteBlock.LinkTo(DataflowBlock.NullTarget <Tuple <string, bool> >());
foreach (string url in urls)
{
isHappySiteBlock.Post(url);
}
isHappySiteBlock.Complete();
addToHappySitesBlock.Completion.Wait();
happySites.ForEach(Console.WriteLine);
}
public void start()
{
var consumerBlock = Consumer("consumer");
var transformBlock = new TransformBlock <int, int>(input =>
{
// if (input == 1) throw new Exception("test problem");
return(input * 3);
});
var producerBlock = new CustomAdvance();
transformBlock.LinkTo(producerBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
for (int i = 0; i < 10; i++)
{
if (!transformBlock.Post(i))
{
Console.WriteLine("Failed Post");
}
}
transformBlock.Complete();
transformBlock.Completion.ContinueWith(p => print_status(p, "transformer"));
consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer"));
Console.Read();
}
public TableLoggerProvider(AzureLoggerSettings settings)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
this.settings = settings;
var cloud = CloudStorageAccount.Parse(settings.ConnectionString);
this.client = cloud.CreateCloudTableClient();
this.table = client.GetTableReference(settings.Table);
this.blobClient = cloud.CreateCloudBlobClient();
if (!string.IsNullOrEmpty(settings.OverflowContainer))
{
this.container = this.blobClient.GetContainerReference(settings.OverflowContainer);
}
this.batchBlock = new BatchBlock <DynamicTableEntity>(25);
Timer triggerBatchTimer = new Timer((_) => this.batchBlock.TriggerBatch(), null, Timeout.Infinite, Timeout.Infinite);
pipeline = new TransformBlock <DynamicTableEntity, DynamicTableEntity>((value) =>
{
triggerBatchTimer.Change(5000, Timeout.Infinite);
return(value);
});
pipelineEnd = new ActionBlock <DynamicTableEntity[]>(WriteBatch);
pipeline.LinkTo(batchBlock);
batchBlock.LinkTo(pipelineEnd);
this.overflowBlock = new ActionBlock <KeyValuePair <string, string> >(WriteOverflow);
}
public async Task Processing(IEnumerable <string> files, string outputDirectory)
{
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var readingBlock = new TransformBlock <string, string>(
async path => await File.ReadAllTextAsync(path),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _pipelineConfiguration.MaxReadingTasks
});
var processingBlock = new TransformManyBlock <string, TestClassInfo>(
async code => await Task.Run(() => TestGenerator.Generate(code)),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _pipelineConfiguration.MaxTasks
});
var writingBlock = new ActionBlock <TestClassInfo>(
async fI => await File.WriteAllTextAsync(outputDirectory + fI.TestClassName + ".txt", fI.TestClassCode),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _pipelineConfiguration.MaxWritingTasks
});
readingBlock.LinkTo(processingBlock, linkOptions);
processingBlock.LinkTo(writingBlock, linkOptions);
foreach (var file in files)
{
readingBlock.Post(file);
}
readingBlock.Complete();
await writingBlock.Completion;
}
public WebhookDequeuer(WebhookSender webhookSender,
IWebhookEventRepository webhookEventRepository,
IWebhookRepository webhookRepository,
IClock clock,
ISemanticLog log)
{
Guard.NotNull(webhookEventRepository, nameof(webhookEventRepository));
Guard.NotNull(webhookRepository, nameof(webhookRepository));
Guard.NotNull(webhookSender, nameof(webhookSender));
Guard.NotNull(clock, nameof(clock));
Guard.NotNull(log, nameof(log));
this.webhookEventRepository = webhookEventRepository;
this.webhookRepository = webhookRepository;
this.webhookSender = webhookSender;
this.clock = clock;
this.log = log;
requestBlock =
new ActionBlock <IWebhookEventEntity>(MakeRequestAsync,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 32, BoundedCapacity = 32
});
blockBlock =
new TransformBlock <IWebhookEventEntity, IWebhookEventEntity>(x => BlockAsync(x),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, BoundedCapacity = 1
});
blockBlock.LinkTo(requestBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
timer = new CompletionTimer(5000, QueryAsync);
}
static void Main(string[] args)
{
var transformBlock = new TransformBlock <int, string>(n =>
{
if (n == 5)
{
throw new Exception("Something went wrong");
}
// Removed all messages in input queue and goes into faulted state
Console.WriteLine($"Received {n}");
return(n.ToString());
});
var printBlock = new ActionBlock <string>(n => Console.WriteLine(n));
transformBlock.LinkTo(printBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < 10; i++)
{
transformBlock.Post(i);
}
transformBlock.Complete();
try
{
printBlock.Completion.Wait();
}
catch (AggregateException ae)
{
throw ae.Flatten().InnerException;
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
public SaveBitmapBlock(string outputPath, int initialZoomLevel, string fileFormat, ExecutionDataflowBlockOptions options,
IGraphicsApi <TImage> graphics)
{
OutputPath = outputPath;
this.fileFormat = fileFormat;
this.graphics = graphics;
Block = new TransformBlock <ImageInfo <TImage>, IEnumerable <SubChunkData> >(info =>
{
if (info == null)
{
return(null);
}
try
{
/*if (r.Next(100) == 0)
* {
* throw new ArgumentOutOfRangeException("TestError in SaveBitmap");
* }*/
SaveBitmap(initialZoomLevel, info.X, info.Z, info.Image);
ProcessedCount++;
return(info.Cd);
}
catch (Exception ex)
{
Console.WriteLine("Error in SaveBitmapBlock: " + ex.Message);
return(null);
}
finally
{
if (info != null)
{
graphics.ReturnImage(info.Image);
info.Image = null;
}
}
}, options);
}
public static TransformBlock <string, string> CreateSimplePipeline(
Action <bool> resultCallback)
{
var step1 = new TransformBlock <string, string>(
sentence => Utils.FindMostCommon(sentence),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 3,
BoundedCapacity = 5,
}
);
var step2 = new TransformBlock <string, int>(
word => Utils.CountChars(word),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = 13,
}
);
var step3 = new TransformBlock <int, bool>(
length => Utils.IsOdd(length),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 11,
BoundedCapacity = 6,
}
);
var callbackStep = new ActionBlock <bool>(resultCallback);
step1.LinkTo(step2, new DataflowLinkOptions());
step2.LinkTo(step3, new DataflowLinkOptions());
step3.LinkTo(callbackStep);
return(step1);
}
private void InitializeDataflowInternal()
{
_uriPropagatorBlock = new UriPropagatorBlock(
CancellationToken,
_logger,
Events,
ShouldAccept,
EmitHeartbeat,
TaskScheduler.Default);
_getUriContentBlock = new TransformBlock <Uri, Resource>(
GetUriContent,
new ExecutionDataflowBlockOptions
{
CancellationToken = CancellationToken,
EnsureOrdered = false,
TaskScheduler = TaskScheduler.Default,
MaxDegreeOfParallelism = _maxUriProcessingDegreeOfParallelism,
MaxMessagesPerTask = 4
});
_uriPropagatorBlock.LinkTo(
_getUriContentBlock,
new DataflowLinkOptions {
PropagateCompletion = true
});
// ensure failed (null) http requests are discarded
_getUriContentBlock.LinkTo(
DataflowBlock.NullTarget <Resource>(),
new DataflowLinkOptions {
PropagateCompletion = true
},
resource => resource == null);
// allow derived implementations to configure their dataflow
InitializeDataflow();
}
public static ISourceBlock <T> Create <T>(string file, Func <Line, T> transform)
{
var block = new TransformBlock <Line, T>(
transform,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
BoundedCapacity = 128
});
new Thread(() =>
{
var input = block.AsObserver();
try
{
using (var reader = new StreamReader(file))
{
string line;
int lineNr = 0;
var batch = new List <Line>();
while ((line = reader.ReadLine()) != null)
{
input.OnNext(new Line {
Content = line, Number = lineNr++
});
}
input.OnCompleted();
}
}
catch (Exception ex)
{
input.OnError(ex);
}
}).Start();
return(block);
}
public void BuildPipeline()
{
_square = new TransformBlock <int, int>(n =>
{
Console.WriteLine($"N = {n}");
return(n * n);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4
});
_printOdd = new ActionBlock <int>(n =>
{
if (n % 2 == 0)
{
Console.BackgroundColor = ConsoleColor.Red;
}
Console.WriteLine($"Odd number is {n}");
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 8
});
_printEven = new ActionBlock <int>(n =>
{
if (n % 2 == 1)
{
Console.BackgroundColor = ConsoleColor.Red;
}
Console.WriteLine($"Even number is {n}");
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 8
});
}
/// <summary>
/// This method is implemented using the Task Parallel Library (TPL).
//. https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-parallel-library-tpl
/// </summary>
private void GetPublications(List <Tool> tools)
{
Logger.LogDebug("Setting up TPL for downloading tools archives.");
var linkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
var downloader = new TransformBlock <Tool, Tool>(
new Func <Tool, Tool>(Downloader), _downloadExeOptions);
var extractXMLs = new TransformBlock <Tool, Tool>(
new Func <Tool, Tool>(WrapperExtractor), _xmlExtractExeOptions);
var extractPublications = new TransformBlock <Tool, Tool>(
new Func <Tool, Tool>(ExtractPublications), _pubExtractExeOptions);
var cleanup = new ActionBlock <Tool>(
input => { Cleanup(input); });
downloader.LinkTo(extractXMLs, linkOptions);
extractXMLs.LinkTo(extractPublications, linkOptions);
extractPublications.LinkTo(cleanup, linkOptions);
Logger.LogDebug($"Starting to process (download archive, extract XML, extract citation) {tools.Count} tools *asynchronously*.");
foreach (var info in tools)
{
downloader.Post(info);
}
downloader.Complete();
cleanup.Completion.Wait();
Logger.LogDebug($"Serializing extract bibliographies to `{_publicationsFilename}`");
WriteToJson(JsonConvert.SerializeObject(Publications, SerializerSettings), _publicationsFilename);
Logger.LogDebug($"Serialized extracted bibliographies.");
}
static async Task Main(string[] args)
{
Generator testGen = new Generator();
var downloadFile = new TransformBlock <string, string>(async filepath =>
{
return(await File.ReadAllTextAsync(filepath));
});
var generateTest = new TransformManyBlock <string, OutputFileInfo>(async testsFile =>
{
return(await Task.Run(() => testGen.GenerateTestClasses(testsFile).ToArray()));
});
var createOutputFiles = new ActionBlock <OutputFileInfo>(async testFile =>
{
await File.WriteAllTextAsync(outputFilesPath + testFile.fileName, testFile.text);
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadFile.LinkTo(generateTest, linkOptions);
generateTest.LinkTo(createOutputFiles, linkOptions);
string[] filePaths = Directory.GetFiles(inputFilesPath);
foreach (string filePath in filePaths)
{
if (filePath.EndsWith(".cs"))
{
downloadFile.Post(filePath);
}
}
downloadFile.Complete();
createOutputFiles.Completion.Wait();
}
public static void FirstExample()
{
var ReadJsonIn = new TransformBlock <string, string>(async FilePath =>
{
Console.WriteLine("Reading Json In From {0}", FilePath);
return(await File.ReadAllTextAsync(FilePath));
});
var TransformToObject = new TransformBlock <string, Appraisal>(json =>
{
Console.WriteLine("Transforming JSON to Object Type: {0}", typeof(Appraisal));
return(JsonConvert.DeserializeObject <Appraisal>(json));
});
var GenerateEvents = new ActionBlock <Appraisal>(appraisal =>
{
var app = (Appraisal)appraisal;
Console.WriteLine("Client Number: {0} recieved", app.ClientId);
Console.WriteLine("Sending Notification to Lender: {0}", app.Lender);
Console.WriteLine("Sending Notification to Client: {0}", app.ClientName);
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
ReadJsonIn.LinkTo(TransformToObject, linkOptions);
TransformToObject.LinkTo(GenerateEvents, linkOptions);
ReadJsonIn.Post(".\\EntrySchema.json");
ReadJsonIn.Complete();
GenerateEvents.Completion.Wait();
}
private static ITargetBlock <string> GetAggregatorFlow(out Dictionary <string, int> result)
{
var splitter = new TransformBlock <string, KeyValuePair <string, int> >(input =>
{
string[] splitted = input.Split('=');
return(new KeyValuePair <string, int>(splitted[0], int.Parse(splitted[1])));
});
var dict = new Dictionary <string, int>();
var aggregater = new ActionBlock <KeyValuePair <string, int> >(pair =>
{
int oldValue;
dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value;
});
var dataflow = _factory.FromPropagator(splitter)
.LinkToTarget(aggregater)
.Create();
result = dict;
return(dataflow);
}
private static IPropagatorBlock <long, long> Create2CrossNode_1(Func <long> f)
{
var target = new TransformBlock <long, long>(x => f()); // f = LongOperation
var source = new BroadcastBlock <long>(x => x);
// linking
target.LinkTo(source);
// completion
target.Completion.ContinueWith(completion =>
{
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
public void BasicUsageTest()
{
int[] array = new int[10];
var evt = new CountdownEvent(10);
var action = new ActionBlock <int> (i =>
{
array [Math.Abs(i)] = i;
evt.Signal();
});
var block = new TransformBlock <int, int> (i => - i);
block.LinkTo(action);
for (int i = 0; i < array.Length; ++i)
{
Assert.IsTrue(block.Post(i), "Not accepted");
}
evt.Wait();
CollectionAssert.AreEqual(
new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
/// <summary>
/// Add in an enumerable of items.
/// </summary>
/// <param name="collection"></param>
public Task AddAsync(IEnumerable <T> collection)
{
if (collection == null)
{
throw new ArgumentNullException("collection");
}
var produce = new TransformBlock <T, T>(item => item, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
});
var consume = new ActionBlock <T>(action: async obj => await this.AddAsync(obj), dataflowBlockOptions: new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
});
produce.LinkTo(consume);
return(Task.Factory.StartNew(async() => {
collection.AsParallel().ForAll(item => produce.SendAsync(item));
produce.Complete();
await consume.Completion;
}));
}
public ITargetBlock <InternalChannelHandlerContext <TKey, TProtocol> > InitReceive()
{
ExecutionDataflowBlockOptions options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
};
var upPakc = new TransformManyBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(UnPack, options);
var decoder = new TransformBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(Decoder, options);
var deserialize = new TransformBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(Deserialize, options);
var messageHandle = new ActionBlock <InternalChannelHandlerContext <TKey, TProtocol> >(MessageHandle, options);
var failHandle = new ActionBlock <InternalChannelHandlerContext <TKey, TProtocol> >(FailHandle, options);
upPakc.LinkTo(decoder, context => context.PipeStatus == PipeStatus.Success);
upPakc.LinkTo(failHandle);
decoder.LinkTo(deserialize, context => context.PipeStatus == PipeStatus.Success);
decoder.LinkTo(failHandle);
deserialize.LinkTo(messageHandle, context => context.PipeStatus == PipeStatus.Success);
deserialize.LinkTo(failHandle);
return(upPakc);
}
private static void Example1() // NOTE: Синхронный прием данных
{
Func <int, int> fn = n =>
{
Thread.Sleep(TimeSpan.FromSeconds(1));
return(n * n);
};
var transformBlock = new TransformBlock <int, int>(fn);
for (var i = 0; i < 10; i++)
{
transformBlock.Post(i);
}
for (var i = 0; i < 10; i++)
{
var result = transformBlock.Receive();
Console.WriteLine(result);
}
Console.WriteLine("Done");
}
public async Task TransformToAction()
{
var t = new TransformBlock <int, int>(i => i * 2);
int completedCount = 0;
int prev = -2;
var c = new ActionBlock <int>(i =>
{
completedCount++;
Assert.Equal(expected: i, actual: prev + 2);
prev = i;
});
t.LinkTo(c, new DataflowLinkOptions {
PropagateCompletion = true
});
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.True(completedCount == Iterations);
}
public void CreatePipeline()
{
var options = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 2,
MaxDegreeOfParallelism = 10,
EnsureOrdered = true
};
buffer = new BufferBlock <Message>();
xForm1 = new TransformBlock <Message, Message>(x => {
Console.WriteLine($"{DateTime.Now.TimeOfDay} - Started Id: {x.Id}");
Task.Delay(rnd.Next(1000, 3000)).Wait();
Console.WriteLine($"{DateTime.Now.TimeOfDay} - Finished Id: {x.Id}");
return(x);
}, options);
action = new ActionBlock <Message>(x => {
Console.WriteLine($"{DateTime.Now.TimeOfDay} - Output Id: {x.Id} Value: {x.Value}");
//this delay will cause the messages to be unordered
Task.Delay(rnd.Next(1000, 3000)).Wait();
OutputMessages.Add(x);
}, options);
}
