public static ITargetBlock<string> SetupPipeline()
{
var fileNamesForPath = new TransformBlock<string, IEnumerable<string>>(
path =>
{
return GetFileNames(path);
});
var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
fileNames =>
{
return LoadLines(fileNames);
});
var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
lines2 =>
{
return GetWords(lines2);
});
var display = new ActionBlock<IEnumerable<string>>(
coll =>
{
foreach (var s in coll)
{
WriteLine(s);
}
});
fileNamesForPath.LinkTo(lines);
lines.LinkTo(words);
words.LinkTo(display);
return fileNamesForPath;
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
_waitHandle.Signal();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock);
transformBlock.LinkTo(actionBlock);
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
_waitHandle.Wait();
}
public async Task TransformThroughDiscardingFilterToAction()
{
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }, i => i % 2 == 0);
t.LinkTo(DataflowBlock.NullTarget<int>());
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / 2, actual: completedCount);
}
public void Run()
{
var cts = new CancellationTokenSource();
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>(new DataflowBlockOptions { CancellationToken = cts.Token });
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Scheduling cancellation after 5 seconds.");
cts.CancelAfter(TimeSpan.FromSeconds(5));
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1, CancellationToken = cts.Token });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10, CancellationToken = cts.Token });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
var t1 = bufferBlock.Completion.ContinueWith(t => Console.WriteLine("Buffer block status: {0}", t.Status));
var t2 = actionBlock.Completion.ContinueWith(t => Console.WriteLine("Action block status: {0}", t.Status));
Console.WriteLine("Waiting for the network to finish.");
Task.WaitAll(t1, t2);
}
public void Start()
{
var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink);
var source = new BufferBlock<GetPageMessage>();
var linkOptions = new DataflowLinkOptions {PropagateCompletion = false};
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var worker = new TransformBlock<GetPageMessage, PageResultMessage>(
(Func<GetPageMessage, PageResultMessage>)Worker, options);
source.LinkTo(worker, linkOptions);
worker.LinkTo(sink, linkOptions);
}
foreach (var url in UrlList.Urls)
{
source.Post(new GetPageMessage{ Url = url });
}
source.Complete();
sink.Completion.Wait();
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Signaling completion to the source block.");
bufferBlock.Complete();
Console.WriteLine("Done.");
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(200);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(500);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
Console.WriteLine("Waiting for the completion to be propagated through the network...");
actionBlock.Completion.ContinueWith(t =>
{
Console.WriteLine("Finished processing.");
Console.WriteLine(string.Format("Completion status: {0}.", t.Status));
}).Wait();
}
public static async Task ValidateBlockAsync(ICoreStorage coreStorage, ICoreRules rules, Chain newChain, ISourceBlock<ValidatableTx> validatableTxes, CancellationToken cancelToken = default(CancellationToken))
{
// tally transactions
object finalTally = null;
var txTallier = new TransformBlock<ValidatableTx, ValidatableTx>(
validatableTx =>
{
var runningTally = finalTally;
rules.TallyTransaction(newChain, validatableTx, ref runningTally);
finalTally = runningTally;
return validatableTx;
});
validatableTxes.LinkTo(txTallier, new DataflowLinkOptions { PropagateCompletion = true });
// validate transactions
var txValidator = InitTxValidator(rules, newChain, cancelToken);
// begin feeding the tx validator
txTallier.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true });
// validate scripts
var scriptValidator = InitScriptValidator(rules, newChain, cancelToken);
// begin feeding the script validator
txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true });
//TODO
await PipelineCompletion.Create(
new Task[] { },
new IDataflowBlock[] { validatableTxes, txTallier, txValidator, scriptValidator });
// validate overall block
rules.PostValidateBlock(newChain, finalTally);
}
public MessagePipeline()
{
linkOptions = new DataflowLinkOptions { PropagateCompletion = true };
buildMessage = new TransformBlock<object, Messaging.Message>(
x => {
Console.WriteLine("buildMessage| message: {0}", x);
return new Messaging.Message { Body = x };
});
logMessage = new TransformBlock<Messaging.Message, Messaging.Message>
(x => {
Console.WriteLine("logMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body);
return x;
});
sendMessage = new TransformBlock<Messaging.Message, Messaging.Message>(
x => {
Console.WriteLine("sendMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body);
return x;
});
buildMessage.LinkTo(logMessage, linkOptions);
logMessage.LinkTo(sendMessage, linkOptions);
}
/// <summary>
/// Search Asynchrony many extension in all of Fixed and Removable Disks.
/// </summary>
/// <param name="targetExtensions">Some file extensions for use search pattern.</param>
/// <example>
/// FileExtension example:
/// {".jpg", 646546Byte, 646Byte}
/// {".pdf", 25464645546Byte, 60000Byte}
/// </example>
/// <returns>A sorted list of detected files</returns>
public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS)
{
return await Task.Run(() =>
{
searchComplete = false;
//
Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---"));
List<FileInfo> _result = new List<FileInfo>();
//
// Find specific folders from windows drives instead of the total drive.
//
FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation();
//
// Set Data-flow
//
TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir =>
{
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchBeginning,
"Searching {0} ...", dir.FullName));
List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS);
Reporter(dir, new ReportEventArgs("DiskProbing",
ReportCodes.TheSearchCompleted,
"The Search {0} was completed!", dir.FullName));
return res;
}, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount });
ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst));
//
// Search specific folders from windows drives instead of the total drive.
//
try
{
TB.LinkTo(AB);
ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount };
var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir));
TB.Complete();
TB.Completion.Wait();
}
catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); }
searchComplete = true;
Reporter("DiskProbing", new ReportEventArgs("DiskProbing",
ReportCodes.DiskProbingFinished,
"---{Search Disks Finished}---"));
LastSearchResult = _result;
return _result;
});
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
static void Main(string[] args)
{
string s =
"http://cn.bing.com/search?q=MD5CryptoServiceProvider+slow&qs=n&pq=md5cryptoserviceprovider+slow&sc=0-25&sp=-1&sk=&cvid=67d40cbd8c424d55a3db83e6e9868267&first=51&FORM=PERE4";
using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider())
{
byte[] inBytes = Encoding.UTF8.GetBytes(s);
var bytes = md5.ComputeHash(inBytes);
Console.WriteLine(bytes.Length);
}
var splitter = new TransformBlock<string, KeyValuePair<string, int>>(
input =>
{
var splitted = input.Split('=');
return new KeyValuePair<string, int>(splitted[0], int.Parse(splitted[1]));
});
var dict = new Dictionary<string, int>();
var aggregater = new ActionBlock<KeyValuePair<string, int>>(
pair =>
{
int oldValue;
dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value;
});
splitter.LinkTo(aggregater, new DataflowLinkOptions() { PropagateCompletion = true});
splitter.Post("a=1");
splitter.Post("b=2");
splitter.Post("a=5");
splitter.Complete();
aggregater.Completion.Wait();
Console.WriteLine("sum(a) = {0}", dict["a"]); //prints sum(a) = 6
//CalcAsync().Wait();
//SlowFlowAsync().Wait();
//FailDemoAsync().Wait();
//TransformAndLinkDemo().Wait();
//LinkLeftToDemo().Wait();
//CircularFlowAutoComplete().Wait();
//RecorderDemo().Wait();
BulkInserterDemo().Wait();
//BulkInserterDemo2().Wait();
//BroadcasterDemo().Wait();
//MyLoggerDemo().Wait();
//ETLLookupDemo().Wait();
}
/// <summary>
/// Initials the transmitter asynchronous.
/// Check the server and then database existence and ...
/// </summary>
public static async Task InitialTransmitterAsync()
{
await ServerValidatorAsync();
ErrorListenerTransformBlock = new TransformBlock<ProxyError, Tuple<ProxyError, bool>>(
async (e) => await TransmitOneError(e),
new ExecutionDataflowBlockOptions()
{
MaxMessagesPerTask = 1,
MaxDegreeOfParallelism = 1
});
ErrorListenerTransformBlock.LinkTo(CacheController.AcknowledgeActionBlock);
}
public void DeferredUsageTest ()
{
int[] array = new int[10];
var action = new ActionBlock<int> (i => array[Math.Abs (i)] = i);
var block = new TransformBlock<int, int> (i => -i);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
Thread.Sleep (300);
block.LinkTo (action);
Thread.Sleep (100);
CollectionAssert.AreEqual (new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public void BasicUsageTest ()
{
int[] array = new int[10];
var evt = new ManualResetEventSlim (false);
ActionBlock<int> action = new ActionBlock<int> ((i) => { array[Math.Abs (i)] = i; evt.Set (); });
TransformBlock<int, int> block = new TransformBlock<int, int> (i => -i);
block.LinkTo (action);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
evt.Wait ();
CollectionAssert.AreEqual (new int[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public Statsd(string serviceName = null)
{
_log.Info("statsd.net starting.");
_tokenSource = new CancellationTokenSource();
_shutdownComplete = new ManualResetEvent(false);
SuperCheapIOC.Add(_log);
var systemInfoService = new SystemInfoService();
SuperCheapIOC.Add(systemInfoService as ISystemInfoService);
serviceName = serviceName ?? systemInfoService.HostName;
var systemMetricsService = new SystemMetricsService("statsdnet", serviceName);
SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService);
/**
* The flow is:
* Listeners ->
* Message Parser ->
* router ->
* Aggregator ->
* Broadcaster ->
* Backends
*/
// Initialise the core blocks
_router = new StatsdMessageRouterBlock();
_messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token,
SuperCheapIOC.Resolve<ISystemMetricsService>(),
_log);
_messageParser.LinkTo(_router);
_messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () =>
{
_log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster.");
_messageBroadcaster.Complete();
});
_messageBroadcaster = new BroadcastBlock<Bucket>(Bucket.Clone);
_messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () =>
{
_log.Info("MessageBroadcaster: Completion signaled. Notifying all backends.");
_backends.ForEach(q => q.Complete());
});
// Add the broadcaster to the IOC container
SuperCheapIOC.Add<BroadcastBlock<Bucket>>(_messageBroadcaster);
systemMetricsService.SetTarget(_messageBroadcaster);
_backends = new List<IBackend>();
_listeners = new List<IListener>();
}
public void ProcessFile(string fileName)
{
var inputBlock = new BufferBlock<string>();
var readBlock = new TransformBlock<string, string>(
(input) =>
{
Console.WriteLine("Loading " + input);
return File.ReadAllText(input);
});
var compileBlock = new TransformBlock<string, string>(
(input) =>
{
Console.WriteLine("Processing...");
var client = new WebClient();
client.Headers.Add("content-type", "application/x-www-form-urlencoded");
string apiData = string.Format(Data, HttpUtility.UrlEncode(input));
return client.UploadString(Url, apiData);
});
var convertBlock = new TransformBlock<string, XDocument>(
(input) =>
{
Console.WriteLine("Converting to XDocument...");
var xml = XDocument.Parse(input);
return xml;
});
var outputBlock = new ActionBlock<XDocument>(
(input) =>
{
Console.WriteLine("Writing compressed-" + fileName);
XElement compiledCode = input.Element("compilationResult").Element("compiledCode");
File.WriteAllText("compressed-" + fileName, compiledCode.Value);
});
inputBlock.LinkTo(readBlock, new DataflowLinkOptions { PropagateCompletion = true });
readBlock.LinkTo(compileBlock, new DataflowLinkOptions { PropagateCompletion = true });
compileBlock.LinkTo(convertBlock, new DataflowLinkOptions { PropagateCompletion = true });
convertBlock.LinkTo(outputBlock, new DataflowLinkOptions { PropagateCompletion = true });
inputBlock.Post(fileName);
inputBlock.Complete();
outputBlock.Completion.Wait();
}
internal static bool TransformThroughFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => true);
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
public static void Example1()
{
var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 };
ActionBlock<int> a = new ActionBlock<int>(i =>
{
Thread.Sleep(500);
Console.WriteLine(i);
}, conf);
TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3);
t.LinkTo(a);
for (int i = 0; i < 12; i++)
{
t.Post(i);
}
}
public void Run()
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var tb = new TransformBlock<int, int>(i => i * 2);
var ab = new ActionBlock<int>(i => this.Compute(i), options);
tb.LinkTo(ab);
for (var i = 0; i < 10; i++)
{
tb.Post(i);
}
tb.Complete();
tb.Completion.Wait();
Thread.Sleep(500);
}
public async Task TransformToAction()
{
var t = new TransformBlock<int, int>(i => i * 2);
int completedCount = 0;
int prev = -2;
var c = new ActionBlock<int>(i =>
{
completedCount++;
Assert.Equal(expected: i, actual: prev + 2);
prev = i;
});
t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
t.PostRange(0, Iterations);
t.Complete();
await c.Completion;
Assert.True(completedCount == Iterations);
}
public void BasicUsageTest ()
{
int[] array = new int[10];
var evt = new CountdownEvent (10);
var action = new ActionBlock<int> (i =>
{
array [Math.Abs (i)] = i;
evt.Signal ();
});
var block = new TransformBlock<int, int> (i => -i);
block.LinkTo (action);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
evt.Wait ();
CollectionAssert.AreEqual (
new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public void Run()
{
Console.WriteLine("Generating first 10 powers of 2.");
var bufferBlock = new BufferBlock<int>();
Enumerable.Range(1, 10)
.ToList()
.ForEach(i => bufferBlock.Post(i));
var transformBlock = new TransformBlock<int, double>(i =>
{
Console.WriteLine("Raising 2 to the power of {0}.", i);
if (i == 5)
{
Console.WriteLine("32 is so mainstream... Throwing exception...");
throw null;
}
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(500);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
var completion = actionBlock.Completion.ContinueWith(t =>
{
Console.WriteLine("Processing failed.");
Console.WriteLine(t.Exception.Message);
}, TaskContinuationOptions.OnlyOnFaulted);
var options = new DataflowLinkOptions { PropagateCompletion = true };
bufferBlock.LinkTo(transformBlock, options);
transformBlock.LinkTo(actionBlock, options);
completion.Wait();
}
static void Main(string[] args)
{
var multiplyBlock = new TransformBlock<int, int>(value => value * 2);
var subtractBlock = new TransformBlock<int, int>(value => value - 2);
var displayBlock = new ActionBlock<int>(value => Console.WriteLine(value));
// multiplyBlock ==> subtractBlock ==> displayBlock
var linkOptions = new DataflowLinkOptions { PropagateCompletion = true };
multiplyBlock.LinkTo(subtractBlock, linkOptions);
subtractBlock.LinkTo(displayBlock, linkOptions);
// Put data in the first block (multiplyBlock)
foreach (var i in Enumerable.Range(0, 10))
multiplyBlock.Post(i);
// Mark it as complete. Completion will propagate because of the link options.
multiplyBlock.Complete();
// Wait for the last block (displayBlock) to complete.
displayBlock.Completion.Wait();
Console.ReadKey();
}
public async static void TData()
{
var multiplyBlock = new TransformBlock<int, int>(item =>
{
var res = item * 2;
Console.WriteLine("{0} * 2 = {1}", item, res);
return res;
});
var divideBlock = new TransformBlock<int, int>(item =>
{
var res = item / 2;
Console.WriteLine("{0} / 2 = {1}", item, res);
return res;
});
multiplyBlock.LinkTo(divideBlock);
multiplyBlock.Post(2);
multiplyBlock.Complete();
await divideBlock.Completion;
}
private static void Main(string[] args)
{
var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num));
var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 });
var finishingBlock = new ActionBlock<string>(str =>
{
writerBlock.Completion.Wait();
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str);
});
generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true});
writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true });
for (var i = 1; i <= 3; i++)
{
Console.WriteLine("Posted " + i*10);
generatorBlock.Post(i*10);
}
generatorBlock.Complete();
finishingBlock.Completion.Wait();
Console.WriteLine("Pipeline is finished");
Console.ReadKey();
}
public static Task GenerateAnnotatedPathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<AnnotatedPath[]> filePathTargetBlock, CancellationToken cancellationToken)
{
var shuffleBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>(
filenames =>
{
// Sequential names tend to fall into the same AWS S3 partition, so we
// shuffle things around.
RandomUtil.Shuffle(filenames);
return filenames;
}, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });
shuffleBlock.LinkTo(filePathTargetBlock, new DataflowLinkOptions { PropagateCompletion = true });
var batcher = new BatchBlock<AnnotatedPath>(2048, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken });
batcher.LinkTo(shuffleBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
return PostAllFilePathsAsync(paths, filePredicate, batcher, cancellationToken);
}
public IDisposable LinkTo(ITargetBlock <OutputFile> target, DataflowLinkOptions linkOptions)
{
return(_generateTestClassBlock.LinkTo(target, linkOptions));
}
async static Task ProcessAsynchronously()
{
var cts = new CancellationTokenSource();
Random _rnd = new Random(DateTime.Now.Millisecond);
Task.Run(() =>
{
if (Console.ReadKey().KeyChar == 'c')
{
cts.Cancel();
}
}, cts.Token);
// BufferBlock：将元素传给流中的下一个块
// BoundedCapacity：指定其容量，超过时不再接受新元素，直到一个现有元素被传递给下一个块
var inputBlock = new BufferBlock <int>(new DataflowBlockOptions {
BoundedCapacity = 5, CancellationToken = cts.Token
});
// TransformBlock：用于数据转换步骤
// MaxDegreeOfParallelism：通过该选项指定最大工作者线程数
// 将int转换为decimal
var convertToDecimalBlock = new TransformBlock <int, decimal>(n =>
{
decimal result = Convert.ToDecimal(n * 100);
Console.WriteLine($"Decimal Converter sent {result} to the next stage on {Thread.CurrentThread.ManagedThreadId}");
Thread.Sleep(TimeSpan.FromMilliseconds(_rnd.Next(200)));
return(result);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
// 将decimal转换为string
var stringifyBlock = new TransformBlock <decimal, string>(n =>
{
string result = $"--{n.ToString("C", CultureInfo.GetCultureInfo("en-us"))}--";
Console.WriteLine($"String Formatter sent {result} to the next stage on {Thread.CurrentThread.ManagedThreadId}");
Thread.Sleep(TimeSpan.FromMilliseconds(_rnd.Next(200)));
return(result);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
// ActionBlock:对每个传入的元素运行一个指定的操作
var outputBlock = new ActionBlock <string>(s => {
Console.WriteLine($"The final result is {s} on thread id {Thread.CurrentThread.ManagedThreadId}");
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
// 通过LinkTo方法将这些块连接到一起
// PropagateCompletion = true : 当前步骤完成时，自动将结果和异常传播到下一个阶段
inputBlock.LinkTo(convertToDecimalBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
convertToDecimalBlock.LinkTo(stringifyBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
stringifyBlock.LinkTo(outputBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
try
{
// 向块中添加项
Parallel.For(0, 20, new ParallelOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
}, i => {
Console.WriteLine($"added {i} to source data on thread id {Thread.CurrentThread.ManagedThreadId}");
inputBlock.SendAsync(i).GetAwaiter().GetResult();
});
// 添加完成后需要调用Complete方法
inputBlock.Complete();
// 等待最后的块完成
await outputBlock.Completion;
Console.WriteLine("Press ENTER to exit.");
}
catch (OperationCanceledException)
{
Console.WriteLine("Operation has been canceled! Press ENTER to exit.");
}
Console.ReadLine();
}
static async Task Main(string[] args)
{
Console.WriteLine("Start");
var option = new ExecutionDataflowBlockOptions {
BoundedCapacity = 2
};
var option1 = new ExecutionDataflowBlockOptions {
BoundedCapacity = 2, MaxDegreeOfParallelism = 3
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var downloader = new TransformBlock <string, byte[]>(async url =>
{
using (var http = new HttpClient())
{
var data = await http.GetByteArrayAsync(url).ConfigureAwait(false);
return(data);
}
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 10
});
var toImage = new TransformBlock <byte[], Image <Rgba32> >(buffer => Image.Load(buffer));
var grayscale = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
image.Mutate(x => x.Grayscale());
return(image);
});
var oilPaint = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+OP ");
image.Mutate(x => x.OilPaint());
Console.Write("-OP ");
return(image);
}, option1);
var pixelate = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+PX ");
image.Mutate(x => x.Pixelate());
Console.Write("-PX ");
return(image);
}, option);
var blur = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+BL ");
image.Mutate(x => x.GaussianBlur());
Console.Write("-BL ");
return(image);
}, option1);
var save = new ActionBlock <Image <Rgba32> >(async image =>
{
using (var f = new FileStream($"{Guid.NewGuid():N}.jpg",
FileMode.Create, FileAccess.Write, FileShare.None, 4096,
FileOptions.Asynchronous))
using (var mem = new MemoryStream())
{
image.SaveAsJpeg(mem);
mem.Position = 0;
await mem.CopyToAsync(f).ConfigureAwait(false);
}
});
toImage.LinkTo(grayscale, linkOptions);
grayscale.LinkTo(oilPaint, linkOptions);
grayscale.LinkTo(blur, linkOptions);
grayscale.LinkTo(pixelate, linkOptions);
pixelate.LinkTo(save);
oilPaint.LinkTo(save);
blur.LinkTo(save);
Task _ = Task.WhenAll(pixelate.Completion, oilPaint.Completion, blur.Completion)
.ContinueWith(c => save.Complete());
for (int i = 0; i < 30; i++)
{
downloader.Post(URL);
}
await Task.Delay(4000).ConfigureAwait(false);
downloader.LinkTo(toImage);
downloader.Complete();
await save.Completion.ConfigureAwait(false);
Console.WriteLine("Done");
Console.ReadKey();
}
public static void Start()
{
const int bc = 1;
// Download a book as a string
var downloadBook = new TransformBlock <string, string>(uri =>
{
Console.WriteLine("Downloading the book...");
return(new WebClient().DownloadString(uri));
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
// splits text into an array of strings.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating list of words...");
// Remove punctuation
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
{
tokens[i] = ' ';
}
}
text = new string(tokens);
return(text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries));
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
// Remove short words and return the count
var filterWordList = new TransformBlock <string[], int>(words =>
{
Console.WriteLine("Counting words...");
var wordList = words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
return(wordList.Count());
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
var printWordCount = new ActionBlock <int>(wordcount =>
{
Console.WriteLine("Found {0} words",
wordcount);
});
downloadBook.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(printWordCount);
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadBook.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printWordCount).Fault(t.Exception);
}
else
{
printWordCount.Complete();
}
});
// Download Origin of Species
downloadBook.Post("http://www.gutenberg.org/files/2009/2009.txt");
downloadBook.Post("http://www.gutenberg.org/files/2010/2010.txt");
downloadBook.Post("http://www.gutenberg.org/files/2011/2011.txt");
// Mark the head of the pipeline as complete.
downloadBook.Complete();
printWordCount.Completion.Wait();
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadLine();
}
public async Task Execute(List <string> filesPath, string outputPath)
{
//Блок чтения из файла
//path,content
var readingBlock = new TransformBlock <string, string>(
async FilePath =>
{
Console.WriteLine("File path: " + FilePath);
using (StreamReader reader = new StreamReader(FilePath))
{
return(await reader.ReadToEndAsync());
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = pipelineConfig.MaxReadingTasks
}
);
//Блок создания теста
//code,tests
var generateTestClass = new TransformManyBlock <string, TestStructure>(
async Code =>
{
Console.WriteLine("Generating tests... ");
return(await TestCreator.Generate(Code));
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = pipelineConfig.MaxProcessingTasks
}
);
//Блок записи теста в файл
//TestStructure
var writeGeneratedFile = new ActionBlock <TestStructure>(
async testClass =>
{
string fullpath = Path.Combine(outputPath, testClass.TestName);
Console.WriteLine("Fullpath " + fullpath);
using (StreamWriter writer = new StreamWriter(fullpath))
{
await writer.WriteAsync(testClass.TestCode);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = pipelineConfig.MaxWritingTasks
}
);
//Successful or unsuccessful completion of one block in the pipeline will
//cause completion of the next block in the pipeline
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
readingBlock.LinkTo(generateTestClass, linkOptions);
generateTestClass.LinkTo(writeGeneratedFile, linkOptions);
foreach (string path in filesPath)
{
readingBlock.Post(path);
}
//Mark the head of the pipeline as complete.
readingBlock.Complete();
await writeGeneratedFile.Completion;
}
/// <summary>
/// <see cref="DualParallelDispatcherRemoteNode{TInput1,TInput2,TOutput1,TOutput2}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="host"><see cref="Host"/></param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public DualParallelDispatcherRemoteNode(
IAppCache persistentCache,
IProgress <double> progress,
Host host,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
ISubject <LinkedItem <TInput1> > item1DispatcherSubject = new Subject <LinkedItem <TInput1> >();
_item1SynchronizedDispatcherSubject = Subject.Synchronize(item1DispatcherSubject);
_item1SynchronizedDispatcherSubjectSubscription = _item1SynchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItem1Async(item.Key.ToString(), item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
ISubject <LinkedItem <TInput2> > item2DispatcherSubject = new Subject <LinkedItem <TInput2> >();
_item2SynchronizedDispatcherSubject = Subject.Synchronize(item2DispatcherSubject);
_item2SynchronizedDispatcherSubjectSubscription = _item2SynchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItem2Async(item.Key.ToString(), item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
var channel = new Channel(host.MachineName, host.Port,
ChannelCredentials.Insecure);
_remoteContract = MagicOnionClient.Create <IRemoteContract <TOutput1, TOutput2> >(channel);
_item1RemoteContract = MagicOnionClient.Create <IOutputItem1RemoteContract <TInput1, TOutput1> >(channel);
_item2RemoteContract = MagicOnionClient.Create <IOutputItem2RemoteContract <TInput2, TOutput2> >(channel);
IRemoteNodeSubject nodeReceiver = new NodeReceiver(_logger);
_remoteNodeHealthSubscription =
nodeReceiver.RemoteNodeHealthSubject.Subscribe(remoteNodeHealth =>
{
NodeMetrics.RemoteNodeHealth = remoteNodeHealth;
});
_nodeHub = StreamingHubClient.Connect <INodeHub, INodeReceiver>(channel, (INodeReceiver)nodeReceiver);
NodeMetrics = new NodeMetrics(Guid.NewGuid());
var item1ProcessSource = new ConcurrentDictionary <Guid, TOutput1>();
var item2ProcessSource = new ConcurrentDictionary <Guid, TOutput2>();
var joinBlock =
new JoinBlock <KeyValuePair <Guid, CancellationTokenSource>, KeyValuePair <Guid, CancellationTokenSource> >(
new GroupingDataflowBlockOptions {
Greedy = false
});
_item1Source =
new TransformBlock <Tuple <Guid, TOutput1, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource>
>(source =>
{
if (!item1ProcessSource.ContainsKey(source.Item1) &&
!item1ProcessSource.TryAdd(source.Item1, source.Item2))
{
_logger.LogError(
$"Could not add item of type {source.Item2.GetType()} and key {source.Item1.ToString()} to the buffer.");
}
return(new KeyValuePair <Guid, CancellationTokenSource>(source.Item1, source.Item3));
});
_item2Source =
new TransformBlock <Tuple <Guid, TOutput2, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource>
>(
source =>
{
if (!item2ProcessSource.ContainsKey(source.Item1) &&
!item2ProcessSource.TryAdd(source.Item1, source.Item2))
{
_logger.LogError(
$"Could not add item of type {source.Item2.GetType()} and key {source.Item1.ToString()} to the buffer.");
}
return(new KeyValuePair <Guid, CancellationTokenSource>(source.Item1, source.Item3));
});
var processBlock = new ActionBlock <Tuple <KeyValuePair <Guid, CancellationTokenSource>,
KeyValuePair <Guid, CancellationTokenSource> > >(
async combined =>
{
var policy = Policy
.Handle <Exception>(ex => !(ex is TaskCanceledException || ex is OperationCanceledException))
.WaitAndRetryAsync(_clusterOptions.RetryAttempt,
retryAttempt =>
TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, sleepDuration, retry, context) =>
{
if (retry >= _clusterOptions.RetryAttempt)
{
_logger.LogError(
$"Could not process item after {retry} retry times: {exception.Message}");
}
});
var policyResult = await policy.ExecuteAndCaptureAsync(async ct =>
{
try
{
if (CpuUsage > _clusterOptions.LimitCpuUsage)
{
var suspensionTime = (CpuUsage - _clusterOptions.LimitCpuUsage) / CpuUsage * 100;
await Task.Delay((int)suspensionTime, ct);
}
if (item1ProcessSource.ContainsKey(combined.Item1.Key) &&
item2ProcessSource.ContainsKey(combined.Item2.Key) &&
item1ProcessSource.TryGetValue(combined.Item1.Key, out var item1) &&
item2ProcessSource.TryGetValue(combined.Item2.Key, out var item2))
{
await _remoteContract.ProcessRemotely(item1, item2, NodeMetrics);
combined.Item1.Value.Cancel();
combined.Item2.Value.Cancel();
}
}
catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
{
_logger.LogTrace("The item process has been cancelled.");
}
}, cts.Token).ConfigureAwait(false);
if (policyResult.Outcome == OutcomeType.Failure)
{
_logger.LogCritical(
policyResult.FinalException != null
? $"Could not process item: {policyResult.FinalException.Message}."
: "An error has occured while processing the item.");
}
if (!item1ProcessSource.TryRemove(combined.Item1.Key, out _))
{
_logger.LogWarning(
$"Could not remove item of key {combined.Item1.ToString()} from the buffer.");
}
if (!item2ProcessSource.TryRemove(combined.Item2.Key, out _))
{
_logger.LogWarning(
$"Could not remove item of key {combined.Item2.ToString()} from the buffer.");
}
});
var options = new DataflowLinkOptions
{
PropagateCompletion = true
};
_item1Source.LinkTo(joinBlock.Target1, options);
_item2Source.LinkTo(joinBlock.Target2, options);
joinBlock.LinkTo(processBlock, options);
}
/// <summary>
/// Получает и кэширует список сотрудников с информацией по отделу из БД employee,
/// получает зарплату сотрудников из веб-сервиса бухгалтерского отдела по коду сотрудника из сервиса кадровиков.
/// </summary>
/// <param name="year"></param>
/// <param name="month"></param>
/// <returns></returns>
public Task <IEnumerable <Employee> > GetEmployeesWithSalaryAsync(int year, int month)
{
// Кэширование объекта или возврат объекта из кэша по ключу.
return(_cache.GetOrCreateAsync(CACHE_KEY + month + year, async(entry) =>
{
// Получение списка всех сотрудников.
var employees = await _employeeRepository.GetEmployeesAsync();
// Параметры, используемые для настройки обработки, выполняемой блоками.
var blockOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
};
// Блок получения кода сотрудника.
var getInnBlock = new TransformBlock <Employee, Employee>(async emp =>
{
#if DEBUG
await Task.Delay(5000); // Эмулируем задержку ответа сервиса.
#endif
emp.BuhCode = await _buhApiClient.GetBuhCodeByInnAsync(emp.Inn);
return emp;
}, blockOptions);
// Блок обработки получения зарплаты сотрудника.
var getSalaryBlock = new ActionBlock <Employee>(async emp =>
{
#if DEBUG
await Task.Delay(5000); // Эмулируем задержку ответа сервиса.
#endif
emp.Salary = await _salaryApiClient.GetSalaryAsync(emp, year, month);
}, blockOptions);
// Параметры, импользуемые для настройки связи между блоками.
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
// Определение связи блоков.
getInnBlock.LinkTo(getSalaryBlock, linkOptions);
// Инициализация обработки.
foreach (var employee in employees)
{
getInnBlock.Post(employee);
}
// Оповещение блока о завершении инициализации обработок.
getInnBlock.Complete();
// Ожидание завершения обработки данных.
getSalaryBlock.Completion.Wait();
// Добавляем сотрудников в кэш.
entry.Value = employees;
// Срок хранения объекта в кэше.
entry.SlidingExpiration = TimeSpan.FromHours(10);
return employees;
}));
}
public Task Execute(CancellationToken token)
{
var options = new DataflowBlockOptions {
CancellationToken = token
};
_buffer = new BufferBlock <long>(options);
var hydrate = new TransformBlock <long, Summoner>(id =>
{
var summoner = _lookup.Hydrate(id);
return(summoner);
}, new ExecutionDataflowBlockOptions {
CancellationToken = token, MaxDegreeOfParallelism = 2
});
var store = new TransformBlock <Summoner, Summoner>(summoner =>
{
if (summoner != null)
{
_storage.Store(summoner);
}
return(summoner);
}, new ExecutionDataflowBlockOptions {
CancellationToken = token, MaxDegreeOfParallelism = 2
});
var crawl = new TransformManyBlock <Summoner, FellowPlayerInfo>(async summoner =>
{
var summoners = new List <FellowPlayerInfo>();
var games = new List <PlayerGameStats>();
if (summoner != null)
{
await _crawler.Crawl(summoner, summoners.Add, games.Add);
}
return(summoners);
}, new ExecutionDataflowBlockOptions {
CancellationToken = token, MaxDegreeOfParallelism = 2
});
var storeNextBatch = new ActionBlock <FellowPlayerInfo>(async info =>
{
if (info != null)
{
var data = await _lookup.Lookup(info.summonerId);
_storage.StoreWhenMissing(data);
}
}, new ExecutionDataflowBlockOptions {
CancellationToken = token, MaxDegreeOfParallelism = 2
});
_buffer.LinkTo(hydrate, new DataflowLinkOptions {
PropagateCompletion = true
});
hydrate.LinkTo(store, new DataflowLinkOptions {
PropagateCompletion = true
});
store.LinkTo(crawl, new DataflowLinkOptions {
PropagateCompletion = true
});
crawl.LinkTo(storeNextBatch, new DataflowLinkOptions {
PropagateCompletion = true
});
return(Task.Run(async() =>
{
while (!token.IsCancellationRequested)
{
try
{
var batch = _producer.Produce((int)TimeSpan.FromDays(1).TotalMinutes, 30);
foreach (var id in batch)
{
await _buffer.SendAsync(id, token);
}
// Start the chain
_buffer.Complete();
// Wait until the chain is complete before iterating again
await storeNextBatch.Completion;
}
catch (Exception ex)
{
throw;
}
}
}, token));
}
Task DoBulkParallel()
{
BufferBlock<string> fileNameStore = new BufferBlock<string>();
int maxParallelism = ImageEngine.NumThreads == 1 ? 1 :
(ImageEngine.NumThreads == -1 ? Environment.ProcessorCount : ImageEngine.NumThreads);
// Define block to perform each conversion
var encoder = new TransformBlock<string, Tuple<byte[], string>>(file =>
{
byte[] data = null;
string filename = Path.GetFileNameWithoutExtension(file) + "." + ImageFormats.GetExtensionOfFormat(SaveFormat);
string path = Path.Combine(BulkUseSourceDestination ? Path.GetDirectoryName(file) : BulkSaveFolder, filename);
path = UsefulThings.General.FindValidNewFileName(path);
using (ImageEngineImage img = new ImageEngineImage(file))
{
try
{
data = img.Save(SaveFormat, SaveMipType, removeAlpha: GeneralRemovingAlpha, customMasks: customMasks);
}
catch (Exception e)
{
BulkConvertFailed.Add(path + " Reason: " + e.ToString());
}
}
BulkProgressValue++;
BulkStatus = $"Converting {BulkProgressValue}/{BulkProgressMax} images.";
return new Tuple<byte[], string>(data, path);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxParallelism, BoundedCapacity = maxParallelism });
// Define block to write converted data to disk
var diskWriter = new ActionBlock<Tuple<byte[], string>>(tuple =>
{
File.WriteAllBytes(tuple.Item2, tuple.Item1);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = maxParallelism }); // Limit to 2 disk write operations at a time, but allow many to be stored in it's buffer.
// Link blocks together
fileNameStore.LinkTo(encoder, new DataflowLinkOptions { PropagateCompletion = true });
encoder.LinkTo(diskWriter, new DataflowLinkOptions { PropagateCompletion = true });
// Begin production
new Action(async () =>
{
foreach (var file in BulkConvertFiles)
await fileNameStore.SendAsync(file);
fileNameStore.Complete();
}).Invoke();
return diskWriter.Completion;
}
private Tuple <ITargetBlock <Type>, Task <DiscoveredParts> > CreateDiscoveryBlockChain(bool typeExplicitlyRequested, IProgress <DiscoveryProgress> progress, CancellationToken cancellationToken)
{
string status = Strings.ScanningMEFAssemblies;
int typesScanned = 0;
var transformBlock = new TransformBlock <Type, object>(
type =>
{
try
{
return(this.CreatePart(type, typeExplicitlyRequested));
}
catch (Exception ex)
{
return(new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.FailureWhileScanningType, type.FullName), ex)
{
AssemblyPath = type.GetTypeInfo().Assembly.Location, ScannedType = type
});
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Debugger.IsAttached ? 1 : Environment.ProcessorCount,
CancellationToken = cancellationToken,
MaxMessagesPerTask = 10,
BoundedCapacity = 100,
});
var parts = ImmutableHashSet.CreateBuilder <ComposablePartDefinition>();
var errors = ImmutableList.CreateBuilder <PartDiscoveryException>();
var aggregatingBlock = new ActionBlock <object>(partOrException =>
{
var part = partOrException as ComposablePartDefinition;
var error = partOrException as PartDiscoveryException;
Debug.Assert(partOrException is Exception == partOrException is PartDiscoveryException, "Wrong exception type returned.");
if (part != null)
{
parts.Add(part);
}
else if (error != null)
{
errors.Add(error);
}
progress.ReportNullSafe(new DiscoveryProgress(++typesScanned, 0, status));
});
transformBlock.LinkTo(aggregatingBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var tcs = new TaskCompletionSource <DiscoveredParts>();
Task.Run(async delegate
{
try
{
await aggregatingBlock.Completion;
tcs.SetResult(new DiscoveredParts(parts.ToImmutable(), errors.ToImmutable()));
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
return(Tuple.Create <ITargetBlock <Type>, Task <DiscoveredParts> >(transformBlock, tcs.Task));
}
public async Task ProcessFiles(string path, IProgress <ProgressReport> progress, RichTextBox txt_description)
{
int totalFilesFound = 0;
int totalFilesRead = 0;
int totalFilesHashed = 0;
int totalFilesUploaded = 0;
DateTime lastReported = DateTime.UtcNow;
void ReportProgress()
{
if (DateTime.UtcNow - lastReported < TimeSpan.FromSeconds(1)) //Try to fire only once a second, but this code is not perfect so you may get a few rapid fire.
{
return;
}
lastReported = DateTime.UtcNow;
var report = new ProgressReport(totalFilesFound, totalFilesRead, totalFilesHashed, totalFilesUploaded);
progress.Report(report);
}
var getFilesBlock = new TransformBlock <string, Dto>(filePath =>
{
var dto = new Dto(filePath, File.ReadAllBytes(filePath));
totalFilesRead++; //safe because single threaded.
return(dto);
});
var hashFilesBlock = new TransformBlock <Dto, Dto>(inDto =>
{
using (var md5 = System.Security.Cryptography.MD5.Create())
{
var hash = md5.ComputeHash(inDto.Data);
var outDto = new Dto(inDto.FilePath, hash, inDto.RelativePath, BitConverter.ToString(hash).Replace("-", ""));
Interlocked.Increment(ref totalFilesHashed); //Need the interlocked due to multithreaded.
ReportProgress();
return(outDto);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, BoundedCapacity = 50
});
var writeToDatabaseBlock = new ActionBlock <Dto>(arg =>
{
//Write to database here.
txt_description.AppendLine($"{arg.RelativePath} ({arg.MD5})");
// Main.AppendText(arg.FilePath);
totalFilesUploaded++;
ReportProgress();
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = 50
});
getFilesBlock.LinkTo(hashFilesBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
hashFilesBlock.LinkTo(writeToDatabaseBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var filePath in Directory.EnumerateFiles(path, "*.*", SearchOption.AllDirectories))
{
await getFilesBlock.SendAsync(filePath).ConfigureAwait(false);
totalFilesFound++;
ReportProgress();
}
getFilesBlock.Complete();
await writeToDatabaseBlock.Completion.ConfigureAwait(false);
ReportProgress();
}
static void Main()
{
// <snippet3>
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words
.Where(word => word.Length > 3)
.Distinct()
.ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
var wordsSet = new HashSet <string>(words);
return(from word in words.AsParallel()
let reverse = new string(word.Reverse().ToArray())
where word != reverse && wordsSet.Contains(reverse)
select word);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
// </snippet3>
// <snippet4>
//
// Connect the dataflow blocks to form a pipeline.
//
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(createWordList, linkOptions);
createWordList.LinkTo(filterWordList, linkOptions);
filterWordList.LinkTo(findReversedWords, linkOptions);
findReversedWords.LinkTo(printReversedWords, linkOptions);
// </snippet4>
// <snippet6>
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// </snippet6>
// <snippet7>
// Mark the head of the pipeline as complete.
downloadString.Complete();
// </snippet7>
// <snippet8>
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
// </snippet8>
}
/// <summary>
/// Opens the configured JobWorker to activate jobs in the given poll interval
/// and handle with the given handler.
/// </summary>
internal void Open()
{
isRunning = true;
var cancellationToken = source.Token;
activateJobsCommand = jobWorkerBuilder.Command;
maxJobsActive = jobWorkerBuilder.Command.Request.MaxJobsToActivate;
// Create options for blocks
var bufferOptions = new DataflowBlockOptions
{
CancellationToken = cancellationToken,
EnsureOrdered = false
};
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = jobWorkerBuilder.ThreadCount,
CancellationToken = cancellationToken,
EnsureOrdered = false
};
// Buffer for polled jobs
var input = new BufferBlock <IJob>(bufferOptions);
// Transformblock to process polled jobs
var transformer = new TransformBlock <IJob, IJob>(async activatedJob =>
{
var jobClient = JobClientWrapper.Wrap(jobWorkerBuilder.JobClient);
try
{
await jobHandler(jobClient, activatedJob);
await TryToAutoCompleteJob(jobClient, activatedJob);
}
catch (Exception exception)
{
await FailActivatedJob(jobClient, activatedJob, cancellationToken, exception);
}
finally
{
jobClient.Reset();
}
return(activatedJob);
},
executionOptions);
// Action block to finalize handled tasks
var output = new ActionBlock <IJob>(activatedJob => { currentJobsActive--; },
executionOptions);
// Link blocks
input.LinkTo(transformer);
transformer.LinkTo(output);
// Start polling
Task.Run(async() =>
{
while (!source.IsCancellationRequested)
{
if (currentJobsActive >= maxJobsActive)
{
await Task.Delay(pollInterval);
continue;
}
var jobCount = maxJobsActive - currentJobsActive;
activateJobsCommand.MaxJobsToActivate(jobCount);
try
{
var response = await activateJobsCommand.Send(null, cancellationToken);
logger?.LogDebug(
"Job worker ({worker}) activated {activatedCount} of {requestCount} successfully.",
activateJobsCommand.Request.Worker,
response.Jobs.Count,
jobCount);
foreach (var job in response.Jobs)
{
await input.SendAsync(job);
currentJobsActive++;
}
}
catch (RpcException rpcException)
{
LogRpcException(rpcException);
}
}
},
cancellationToken).ContinueWith(
t => logger?.LogError(t.Exception, "Job polling failed."),
TaskContinuationOptions.OnlyOnFaulted);
var command = jobWorkerBuilder.Command;
logger?.LogDebug(
"Job worker ({worker}) for job type {type} has been opened.",
command.Request.Worker,
command.Request.Type);
}
public void Start(ISenderCallback callback)
{
_callback = callback;
_sender = new ActionBlock <OutgoingMessageBatch>(SendBatch, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_sender.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_serializing = new ActionBlock <Envelope>(async e =>
{
try
{
e.EnsureData();
await _batching.SendAsync(e);
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error while trying to serialize envelope {e}");
}
},
new ExecutionDataflowBlockOptions
{
CancellationToken = _cancellation,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_serializing.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_batchWriting = new TransformBlock <Envelope[], OutgoingMessageBatch>(
envelopes =>
{
var batch = new OutgoingMessageBatch(Destination, envelopes);
_queued += batch.Messages.Count;
return(batch);
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded, MaxDegreeOfParallelism = 10
});
_batchWriting.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_batchWriting.LinkTo(_sender);
_batching = new BatchingBlock <Envelope>(200, _batchWriting, _cancellation);
_batching.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
}
static async void StartCrawling()
{
if (!Directory.Exists("Images"))
{
Directory.CreateDirectory("Images");
}
try
{
#region Dataflow block Options
var downloaderOptions = new ExecutionDataflowBlockOptions
{
// enforce fairness, after handling n messages
// the block's task will be re-schedule.
// this will give the opportunity for other block
// to actively process there messages (to avoid over subscription
// the Tpl dataflow does not schedule all task at once if the machine
// does not have enough cores)
MaxMessagesPerTask = DOWNLOADER_MAX_MESSAGE_PER_TASK,
// by default Tpl dataflow assign a single task per block,
// but you can control it by using the MaxDegreeOfParallelism
MaxDegreeOfParallelism = DOWNLOADER_MAX_DEGREE_OF_PARALLELISM,
// the size of the block input buffer
BoundedCapacity = DOWNLOADER_BOUNDED_CAPACITY
};
var transformerOptions = new ExecutionDataflowBlockOptions
{
MaxMessagesPerTask = MAX_MESSAGE_PER_TASK,
};
var writerOptions = new ExecutionDataflowBlockOptions
{
// by default Tpl dataflow assign a single task per block,
// but you can control it by using the MaxDegreeOfParallelism
MaxDegreeOfParallelism = WRITER_MAX_DEGREE_OF_PARALLELISM,
// MaxMessagesPerTask = MAX_MESSAGE_PER_TASK,
};
var linkOption = new DataflowLinkOptions {
PropagateCompletion = true
};
#endregion // Dataflow block Options
#region Downloader
var downloader = new TransformBlock <string, HttpContentInfo>( // "text/html, image/jpeg"
async(url) =>
{
try
{
#region Validation
if (_urls.ContainsKey(url))
{
return(null);
}
_urls.TryAdd(url, true);
if (!ShouldContinue(url))
{
return(null);
}
#endregion // Validation
HttpClient client = new HttpClient();
client.Timeout = TimeSpan.FromSeconds(DOWNLOAD_TIMEOUT_SEC);
//Trace.WriteLine("Downloading: " + url);
// using IOCP the thread pool worker thread does return to the pool
HttpResponseMessage response = await client.GetAsync(url);
if (!response.IsSuccessStatusCode)
{
WriteToConsole("Fail to download html: [{0}] \r\n\tStatus Code = {1}", ConsoleColor.Red, url, response.StatusCode);
return(null);
}
HttpContent content = response.Content;
var contentType = content.Headers.ContentType;
string mediaType = contentType.MediaType;
#region Validation
if (contentType == null)
{
WriteToConsole("Unknown content type [{0}]: {1}", ConsoleColor.Gray,
mediaType, url);
return(null);
}
#endregion // Validation
WriteToConsole("Downloaded [{0}]: {1}", ConsoleColor.White,
mediaType, url);
var info = new HttpContentInfo(url, response.Content);
if (!IsMediaType(info, "text/html"))
{
Trace.WriteLine("Downloaded [" + mediaType + "]: " + url);
}
return(info);
}
#region Exception Handling
catch (UriFormatException ex)
{
WriteToConsole("invalid URL", ConsoleColor.Red, ex.Message);
}
catch (WebException ex)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, ex.Message);
}
catch (AggregateException ex)
{
foreach (var exc in ex.Flatten().InnerExceptions)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, exc.Message);
}
}
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
}
#endregion // Exception Handling
return(null);
}, downloaderOptions);
#endregion // Downloader
#region Parser
var parser = new TransformManyBlock <HttpContentInfo, string>(
async contentInfo =>
{
HttpContent content = contentInfo.Content;
// using IOCP the thread pool worker thread does return to the pool
string html = await content.ReadAsStringAsync();
var output = new List <string>();
try
{
var links = _linkRegex.Matches(html);
foreach (Match item in links)
{
var value = item.Value;
//Trace.WriteLine("\t\tPARSED: " + value);
output.Add(value);
}
}
#region Exception Handling
catch (Exception ex)
{
WriteToConsole("Error {0}", ConsoleColor.Red, ex.Message);
}
#endregion // Exception Handling
return(output);
}, transformerOptions);
#endregion // Parser
#region Writer
var writer = new ActionBlock <HttpContentInfo>(async contentInfo =>
{
try
{
HttpContent content = contentInfo.Content;
// using IOCP the thread pool worker thread does return to the pool
using (Stream source = await content.ReadAsStreamAsync())
using (var image = Image.FromStream(source))
{
string fileName = Path.GetFileName(contentInfo.Url);
//Trace.WriteLine("\tWRITTING: " + contentInfo.Url);
#region Validation
if (!_images.TryAdd(fileName, true))
{
return;
}
if (image.Width < MIN_SIZE.Width || image.Height < MIN_SIZE.Height)
{
return;
}
#endregion // Validation
string name = @"Images\" + fileName;
using (Stream dest = OpenWriteAsync(name))
{
source.Position = 0;
// using IOCP the thread pool worker thread does return to the pool
await source.CopyToAsync(dest);
WriteToConsole("{0}: Width:{1}, Height:{2}", ConsoleColor.Yellow,
fileName, image.Width, image.Height);
}
}
}
#region Exception Handling
catch (WebException ex)
{
WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, ex.Message);
}
catch (Exception ex)
{
WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message);
}
#endregion // Exception Handling
}, writerOptions);
#endregion // Writer
var garbageContent = DataflowBlock.NullTarget <HttpContentInfo>();
var garbageUrl = DataflowBlock.NullTarget <string>();
#region LinkTo
////////////////////////////////////////////////////////
// //
// garbage <------- downloader <-------------- //
// / \ | //
// writer parser -------- //
// //
////////////////////////////////////////////////////////
downloader.LinkTo(writer, linkOption, _isImage);
downloader.LinkTo(parser, linkOption, info => info != null);
downloader.LinkTo(garbageContent, linkOption); // fallback (otherwise empty messages will be stack in the block buffer and the block will never complete)
parser.LinkTo(downloader, linkOption, url => !string.IsNullOrEmpty(url));
parser.LinkTo(garbageUrl, linkOption);
#endregion // LinkTo
downloader.Post(URL_CRAWL_TARGET);
Console.WriteLine("Crawling");
Thread.Sleep(COMPLETE_AFTER_SEC * 1000);
#region Complete
downloader.Complete();
#region WriteToConsole ("Try to Complete...")
ConsoleColor color = ConsoleColor.Yellow;
WriteToConsole(
@"Try to Complete (items in the buffer =
downloader: is completed = {0}, input={1} , output={2}
writer: is completed = {3}, input ={4}
parser: is completed = {5}, input={6} , output={7}", color,
downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount,
writer.Completion.IsCompleted, writer.InputCount,
parser.Completion.IsCompleted, parser.InputCount, parser.OutputCount);
#endregion // WriteToConsole ("Try to Complete...")
Task completeAll = Task.WhenAll(
downloader.Completion,
parser.Completion,
writer.Completion);
await Task.Run(async() =>
{
while (!completeAll.IsCompleted)
{
await Task.Delay(2000);
#region WriteToConsole (status)
color = color == ConsoleColor.Magenta ? ConsoleColor.White : ConsoleColor.Yellow;
WriteToConsole(
@"Complete Status (items in the buffer =
downloader: is completed = {0}, input={1} , output={2}
writer: is completed = {3}, input ={4}
parser: is completed = {5}, input={6} , output={7}", color,
downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount,
writer.Completion.IsCompleted, writer.InputCount,
parser.Completion.IsCompleted, parser.InputCount, parser.OutputCount);
}
#endregion // WriteToConsole (status)
});
WriteToConsole("Complete (items in the writer input buffer = {0})", ConsoleColor.Green, writer.InputCount);
#endregion // Complete
}
catch (Exception ex)
{
WriteToConsole("EXCEPTION: {0}", ConsoleColor.DarkRed, ex);
}
}
public SMSOperationWorker()
{
try
{
if (string.IsNullOrEmpty(SP.ep.SendBlockDueTime))
{
DueTime = Convert.ToInt32(SP.ep.SendBlockDueTime);
}
if (string.IsNullOrEmpty(SP.ep.SendBlockBatchMaxNum))
{
BatchMaxNum = Convert.ToInt32(SP.ep.SendBlockBatchMaxNum);
}
this.KeyParam = "";
_logCaches = new BatchBlock <ReceiveMsgStruct>(BatchMaxNum);
mainTB = new TransformBlock <ReceiveMsgStruct[], List <SendMsgStruct> >(array =>
{
try
{
return(EmaySendAction(array));
}
catch (Exception ex)
{
AsyncHelper.RunSync <bool>(() => Manager.Instance.WriteLogFile("EmaySendAction出现异常", ex));
return(new List <SendMsgStruct>());
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
spareTB = new ActionBlock <List <SendMsgStruct> >(array =>
{
try
{
if (array != null && array.Count > 0)
{
CnkiSendAction(array);
}
}
catch (Exception ex)
{
AsyncHelper.RunSync <bool>(() => Manager.Instance.WriteLogFile("CnkiSendAction出现异常", ex));
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
mainTB.LinkTo(spareTB);
_logCaches.LinkTo(mainTB);
triggerBatchTimer = new Timer((obj) => _logCaches.TriggerBatch());
_logCaches.Completion.ContinueWith(delegate { mainTB.Complete(); });
mainTB.Completion.ContinueWith(delegate { spareTB.Complete(); });
}
catch (Exception ex)
{
AsyncHelper.RunSync <bool>(() => Manager.Instance.WriteLogFile("SMSOperationWorker出现异常", ex));
}
}
// Creates the image processing dataflow network and returns the
// head node of the network.
ITargetBlock <string> CreateImageProcessingNetwork()
{
//
// Create the dataflow blocks that form the network.
//
// Create a dataflow block that takes a folder path as input
// and returns a collection of Bitmap objects.
var loadBitmaps = new TransformBlock <string, IEnumerable <Bitmap> >(path =>
{
try
{
return(LoadBitmaps(path));
}
catch (OperationCanceledException)
{
// Handle cancellation by passing the empty collection
// to the next stage of the network.
return(Enumerable.Empty <Bitmap>());
}
});
// Create a dataflow block that takes a collection of Bitmap objects
// and returns a single composite bitmap.
var createCompositeBitmap = new TransformBlock <IEnumerable <Bitmap>, Bitmap>(bitmaps =>
{
try
{
return(CreateCompositeBitmap(bitmaps));
}
catch (OperationCanceledException)
{
// Handle cancellation by passing null to the next stage
// of the network.
return(null);
}
});
// Create a dataflow block that displays the provided bitmap on the form.
var displayCompositeBitmap = new ActionBlock <Bitmap>(bitmap =>
{
// Display the bitmap.
pictureBox1.SizeMode = PictureBoxSizeMode.StretchImage;
pictureBox1.Image = bitmap;
// Enable the user to select another folder.
toolStripButton1.Enabled = true;
toolStripButton2.Enabled = false;
Cursor = DefaultCursor;
},
// Specify a task scheduler from the current synchronization context
// so that the action runs on the UI thread.
new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext()
});
// Create a dataflow block that responds to a cancellation request by
// displaying an image to indicate that the operation is cancelled and
// enables the user to select another folder.
var operationCancelled = new ActionBlock <object>(delegate
{
// Display the error image to indicate that the operation
// was cancelled.
pictureBox1.SizeMode = PictureBoxSizeMode.CenterImage;
pictureBox1.Image = pictureBox1.ErrorImage;
// Enable the user to select another folder.
toolStripButton1.Enabled = true;
toolStripButton2.Enabled = false;
Cursor = DefaultCursor;
},
// Specify a task scheduler from the current synchronization context
// so that the action runs on the UI thread.
new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext()
});
//
// Connect the network.
//
// Link loadBitmaps to createCompositeBitmap.
// The provided predicate ensures that createCompositeBitmap accepts the
// collection of bitmaps only if that collection has at least one member.
loadBitmaps.LinkTo(createCompositeBitmap, bitmaps => bitmaps.Count() > 0);
// Also link loadBitmaps to operationCancelled.
// When createCompositeBitmap rejects the message, loadBitmaps
// offers the message to operationCancelled.
// operationCancelled accepts all messages because we do not provide a
// predicate.
loadBitmaps.LinkTo(operationCancelled);
// Link createCompositeBitmap to displayCompositeBitmap.
// The provided predicate ensures that displayCompositeBitmap accepts the
// bitmap only if it is non-null.
createCompositeBitmap.LinkTo(displayCompositeBitmap, bitmap => bitmap != null);
// Also link createCompositeBitmap to operationCancelled.
// When displayCompositeBitmap rejects the message, createCompositeBitmap
// offers the message to operationCancelled.
// operationCancelled accepts all messages because we do not provide a
// predicate.
createCompositeBitmap.LinkTo(operationCancelled);
// Return the head of the network.
return(loadBitmaps);
}
public void Init()
{
_massMail = new MassMail(Config.BlockSize, Config.UserAgent, Config.ConnectionString, Config.Mode);
_templateCache = new ConcurrentDictionary <long, Lazy <Template> >();
_attachmentCache = new ConcurrentDictionary <long, Lazy <Attach> >();
_dkimSignerCache = new ConcurrentDictionary <string, DkimSigner>();
_domailKeySignerCache = new ConcurrentDictionary <string, DomainKeySigner>();
//Get all private keys
GetDkimSigners();
//*** Create pipeline ***
//Create TransformBlock that gets table of client data and make a list of objects from them.
_parseXmlDataBlock = new TransformBlock <DataTable, List <Mail> >(sendData => ParseXmlData(sendData),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.ParseXmlMaxdop,
BoundedCapacity = Config.ParseXmlBufferSize
});
//Create TransformBlock that gets a list of client objects, send them email, and stores result in DataTable.
_sendEmailsBlock = new TransformBlock <List <Mail>, DataTable>(mails => SendEmails(_massMail, mails),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.SendEmailsMaxdop,
BoundedCapacity = Config.SendEmailsMaxdop
});
//Create BatchBlock that holds several DataTable and then propagates them out as an array.
_batchResultBlock = new BatchBlock <DataTable>(Config.BatchSize,
new GroupingDataflowBlockOptions
{
BoundedCapacity = Config.BatchSize
});
//Create ActionBlock that writes result into DB
_writeResultsBlock = new ActionBlock <DataTable[]>(results => WriteResults(_massMail, results),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
});
//*** Build pipeline ***
// POST --> _parseXmlDataBlock --> _sendEmailsBlock --> _batchResultBlock --> _writeResultsBlock
_parseXmlDataBlock.LinkTo(_sendEmailsBlock);
_sendEmailsBlock.LinkTo(_batchResultBlock);
_batchResultBlock.LinkTo(_writeResultsBlock);
_parseXmlDataBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_sendEmailsBlock).Fault(t.Exception);
}
else
{
_sendEmailsBlock.Complete();
}
});
_sendEmailsBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_batchResultBlock).Fault(t.Exception);
}
else
{
_batchResultBlock.Complete();
}
});
_batchResultBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_writeResultsBlock).Fault(t.Exception);
}
else
{
_writeResultsBlock.Complete();
}
});
}
/// <summary>
/// Load readers from database.
/// </summary>
public void LoadReaders()
{
int maxMfn = Connection.GetMaxMfn(Database);
const int delta = 1000;
DataflowLinkOptions linkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
TransformBlock <string, MarcRecord> parseRecordBlock
= new TransformBlock <string, MarcRecord>
(
line => _ParseRecord(line),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 6,
SingleProducerConstrained = true
}
);
TransformBlock <MarcRecord, ReaderInfo> parseReaderBlock
= new TransformBlock <MarcRecord, ReaderInfo>
(
record => _ParseReader(record),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4
}
);
ActionBlock <ReaderInfo> analyzeReaderBlock
= new ActionBlock <ReaderInfo>
(
reader => _AnalyzeReader(reader),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4
}
);
parseRecordBlock.LinkTo
(
parseReaderBlock,
linkOptions
);
parseReaderBlock.LinkTo
(
analyzeReaderBlock,
linkOptions
);
DateTime today = DateTime.Today;
_debtorManager
= new DebtorManager(Connection)
{
FromDate = today.AddYears(-1),
ToDate = today.AddMonths(-1)
};
_debtorManager.SetupDates();
for (int offset = 1; offset < maxMfn; offset += delta)
{
WriteLine
(
"Загружается: {0} из {1}",
offset - 1,
maxMfn - 1
);
int portion = Math.Min(delta, maxMfn - offset);
int[] list = Enumerable.Range(offset, portion)
.ToArray();
string[] lines = _ReadRawRecords(list);
foreach (string line in lines)
{
parseRecordBlock.Post(line);
}
}
parseRecordBlock.Complete();
analyzeReaderBlock.Completion.Wait();
Readers.CompleteAdding();
Debtors.CompleteAdding();
}
private static async Task <UsageInformation> GetUsageInformation(IEnumerable <FileInfo> files)
{
var result = new UsageInformation();
var multipleInstancesOption = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var singleInstanceOption = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
};
var getApisBlock = new TransformBlock <FileInfo, UsageInformation>(GetApisUsedByAssembly, multipleInstancesOption);
var collectApisBlock = new ActionBlock <UsageInformation>(assemblyInfo =>
{
if (assemblyInfo != null)
{
foreach (var targetFramework in assemblyInfo.TargetFrameworks)
{
result.TargetFrameworks.Add(targetFramework);
}
foreach (var version in assemblyInfo.Versions)
{
result.Versions.Add(version);
}
foreach (var apisByAssembly in assemblyInfo.MemberReferences)
{
if (!result.MemberReferences.TryGetValue(apisByAssembly.Key, out HashSet <string> allApis))
{
allApis = new HashSet <string>();
result.MemberReferences[apisByAssembly.Key] = allApis;
}
foreach (var api in apisByAssembly.Value)
{
allApis.Add(api);
}
}
}
},
singleInstanceOption);
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
getApisBlock.LinkTo(collectApisBlock, linkOptions);
foreach (var file in files)
{
await getApisBlock.SendAsync(file);
}
getApisBlock.Complete();
await Task.WhenAll(getApisBlock.Completion, collectApisBlock.Completion);
return(result);
}
public Task UploadBlobsAsync(AwsManager awsManager, ISourceBlock<Tuple<IFileFingerprint, AnnotatedPath>> uniqueBlobBlock,
IReadOnlyDictionary<string, string> knowObjects, CancellationToken cancellationToken)
{
var blobCount = 0;
var blobTotalSize = 0L;
var builderBlock = new TransformBlock<Tuple<IFileFingerprint, AnnotatedPath>, S3Blobs.IUploadBlobRequest>(
tuple =>
{
string etag;
var exists = knowObjects.TryGetValue(tuple.Item1.Fingerprint.Key(), out etag);
//Debug.WriteLine($"{tuple.Item1.FullFilePath} {(exists ? "already exists" : "scheduled for upload")}");
if (exists)
{
// We can't check multipart uploads this way since we don't know the size
// of the individual parts.
if (etag.Contains("-"))
{
Debug.WriteLine($"{tuple.Item1.FullFilePath} is a multi-part upload with ETag {etag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}");
return null;
}
var expectedETag = tuple.Item1.Fingerprint.S3ETag();
if (string.Equals(expectedETag, etag, StringComparison.InvariantCultureIgnoreCase))
return null;
Console.WriteLine($"ERROR: {tuple.Item1.FullFilePath} tag mismatch {etag}, expected {expectedETag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}");
}
var request = awsManager.BuildUploadBlobRequest(tuple);
if (null == request)
return null;
Interlocked.Increment(ref blobCount);
Interlocked.Add(ref blobTotalSize, request.FileFingerprint.Fingerprint.Size);
return request;
},
new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });
var uploader = new ActionBlock<S3Blobs.IUploadBlobRequest>(
blob => UploadBlobAsync(awsManager, blob, cancellationToken),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
CancellationToken = cancellationToken
});
builderBlock.LinkTo(uploader, new DataflowLinkOptions { PropagateCompletion = true }, r => null != r);
builderBlock.LinkTo(DataflowBlock.NullTarget<S3Blobs.IUploadBlobRequest>());
uniqueBlobBlock.LinkTo(builderBlock, new DataflowLinkOptions { PropagateCompletion = true });
var tasks = new List<Task>();
#if DEBUG
var uploadDoneTask = uploader.Completion
.ContinueWith(
_ => Debug.WriteLine($"Done uploading blobs: {blobCount} items {SizeConversion.BytesToGiB(blobTotalSize):F2}GiB"), cancellationToken);
tasks.Add(uploadDoneTask);
#endif
tasks.Add(uploader.Completion);
return Task.WhenAll(tasks);
}
static void Main(string[] args)
{
// Test data
args = new string[] { "C:\\Development\\SonDar\\Paragon\\GuardAnalyzer", "Preview", "Default", "Example997*.cs" };
// arg0 : path to folder
string pathToStartFolder = args[0];
// arg1 : work mode
WorkMode mode;
if (!Enum.TryParse(args[1], out mode))
{
throw new Exception("Unknown Mode [args[1] = " + mode + "]");
}
// arg2 : changeModel file (Optional)
string path = Directory.GetCurrentDirectory() + "\\changes.txt";
if (args.Length > 2 && !args[2].Equals("Default"))
{
path = args[2];
}
// arg3 - wirdcard(Optional)
string wildcard = "*.cs";
if (args.Length > 3)
{
wildcard = args[3];
}
// Blocks list
TransformBlock <string, ArrayList> fileListBuilder = new TransformBlock <string, ArrayList>
(domainFolderName => (new FileListBuilder()).ParseDirectory(domainFolderName, wildcard));
TransformBlock <ArrayList, ChangeModel> guardAnalyzer = new TransformBlock <ArrayList, ChangeModel>
(fileList => (new GuardAnalyzer()).Prepare(fileList));
TransformBlock <string, ChangeModel> loader = new TransformBlock <string, ChangeModel>
(fileName => ChangeModel.Load(fileName));
TransformBlock <ChangeModel, ChangeModel> printer = new TransformBlock <ChangeModel, ChangeModel>
(model => model.Print());
ActionBlock <ChangeModel> writer = new ActionBlock <ChangeModel>(tempModel => tempModel.Save(path));
ActionBlock <ChangeModel> commiter = new ActionBlock <ChangeModel>(tempModel => (new GuardAnalyzer()).Commit(tempModel));
//Start
// Preview - fileListBuilder->guardAnalyzer->printer->writer
// Commit - loader->printer->commiter
// Forse - fileListBuilder->guardAnalyzer->printer->commiter
Logger.Log("Preview : ");
if (mode == WorkMode.Preview || mode == WorkMode.Force)
{
fileListBuilder.LinkTo(guardAnalyzer);
fileListBuilder.Completion.ContinueWith(task => guardAnalyzer.Complete());
guardAnalyzer.LinkTo(printer);
guardAnalyzer.Completion.ContinueWith(task => printer.Complete());
if (mode != WorkMode.Force)
{
printer.LinkTo(writer);
printer.Completion.ContinueWith(task => writer.Complete());
fileListBuilder.Post(pathToStartFolder);
fileListBuilder.Complete();
writer.Completion.Wait();
}
}
if (mode == WorkMode.Commit || mode == WorkMode.Force)
{
if (mode != WorkMode.Force)
{
loader.LinkTo(printer);
loader.Completion.ContinueWith(task => printer.Complete());
}
printer.LinkTo(commiter);
printer.Completion.ContinueWith(task => commiter.Complete());
if (mode != WorkMode.Force)
{
loader.Post(path);
loader.Complete();
}
else
{
fileListBuilder.Post(pathToStartFolder);
fileListBuilder.Complete();
}
commiter.Completion.Wait();
}
Console.ReadKey();
}
static void SweepCommandLine(string outputFile, string vwExe, string dataFile, string outModelDir, int numProcs = 30)
{
var bags = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--bag " + a);
var softmaxes = new[] { 0, 1, 2, 4, 8, 16 }.Select(a => "--softmax --lambda " + a);
var epsilons = new[] { .2f }.Select(a => "--epsilon " + a);
var covers = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--cover " + a);
var arguments = Util.Expand(
epsilons.Union(bags).Union(softmaxes).Union(covers),
new[] { "--cb_type ips", "--cb_type mtr", "--cb_type dr" },
new[] { "--marginal KG", "--marginal G", "--marginal K", "" },
new[] { 0.0002, 0.005, 0.01, 0.1 }.Select(l => string.Format(CultureInfo.InvariantCulture, "-l {0}", l))
)
.Select((a, i) => $"--cb_explore_adf --ignore B --ignore C --ignore D --ignore E --ignore F --ignore H --ignore R -b 18 --power_t 0 {a} -d {dataFile} -f {outModelDir}\\{i}.model -c")
.ToList();
Directory.CreateDirectory(outModelDir);
File.WriteAllLines(outputFile, arguments);
int numFinishedProcessing = 0;
var inputBlock = new TransformBlock <int, int>(i => i);
var processBlock = new ActionBlock <int>(i =>
{
var startTime = DateTime.UtcNow;
var p = Process.Start(new ProcessStartInfo
{
FileName = vwExe,
Arguments = arguments[i],
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
});
string output = p.StandardOutput.ReadToEnd();
string error = p.StandardError.ReadToEnd();
File.WriteAllText($"{outModelDir}\\{i}.output", $"{startTime}\r\n{arguments[i]}\r\n{output}\r\n{error}");
p.WaitForExit();
int numFinished = Interlocked.Increment(ref numFinishedProcessing);
Console.WriteLine($"Finished: {numFinishedProcessing} / {arguments.Count}");
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = numProcs, BoundedCapacity = numProcs
});
inputBlock.LinkTo(processBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var input = inputBlock.AsObserver();
for (int i = 0; i < arguments.Count; i++)
{
input.OnNext(i);
}
input.OnCompleted();
processBlock.Completion.Wait();
}
public void InitializeDataflow()
{
_config = GlobalClaimWakeUp.Container.Resolve<ClaimProcessorConfiguration>();
Logger.Info("======================Initializing Dataflow=================================");
GlobalClaimWakeUp.PostedClaims.Clear();
if (_config.ReopeningClaims)
{
GlobalClaimWakeUp.GeniusXPolicyState.Clear();
GlobalClaimWakeUp.NameReferences.Clear();
GlobalClaimWakeUp.PostedClaims.Clear();
}
_attachClaimToPolicy = new TransformBlock<Claim, Claim>(
c => AttachClaimToPolicyBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_executeExcessAndDeductibles = new TransformBlock<Claim, Claim>(
c => ExcessAndDeductiblesBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_createTransferNameRequests = new TransformBlock<Claim, Claim>(
c => CreateNameTransferRequestsBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
// We don't need to create a review task for the service
//_createClaimReviewTask = new TransformBlock<Claim, Claim>(
// c => CreateReviewTaskBlock.Execute(c),
// new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_createTransferClaimRequest = new TransformBlock<Claim, Claim>(
c => CreateClaimTransferRequestsBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_clearLocks = new TransformBlock<Claim, Claim>(
c => ClearLocksBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_validateClaim = new TransformBlock<Claim, Claim>(
c => ValidateClaimBlock.Execute(c),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_setCustomCode18 = new TransformBlock<Claim, Claim>(
c =>
{
SetCustomCode18Block.Execute(c);
UpdateMigrationStatus(c);
return c;
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
_logClaimState = new ActionBlock<Claim>(
c =>
{
Logger.InfoFormat("Claim status is\r\n{0}", JObject.FromObject(c));
});
//_attachClaimToPolicy.LinkTo(_createClaimReviewTask, new DataflowLinkOptions { PropagateCompletion = true });
//_createClaimReviewTask.LinkTo(_executeExcessAndDeductibles, new DataflowLinkOptions { PropagateCompletion = true });
// Created next line to avoid the review task creation.
_attachClaimToPolicy.LinkTo(_executeExcessAndDeductibles, new DataflowLinkOptions { PropagateCompletion = true });
_executeExcessAndDeductibles.LinkTo(_createTransferClaimRequest, new DataflowLinkOptions { PropagateCompletion = true });
_createTransferClaimRequest.LinkTo(_createTransferNameRequests, new DataflowLinkOptions { PropagateCompletion = true });
_createTransferNameRequests.LinkTo(_validateClaim, new DataflowLinkOptions { PropagateCompletion = true });
_validateClaim.LinkTo(_clearLocks, new DataflowLinkOptions { PropagateCompletion = true });
_clearLocks.LinkTo(_setCustomCode18, new DataflowLinkOptions { PropagateCompletion = true });
_setCustomCode18.LinkTo(_logClaimState, new DataflowLinkOptions {PropagateCompletion = true});
}
private static void LinearizeDatabase(Args arguments)
{
// and a couple of checks here
Contract.Requires(arguments.InputDirectory != null, "You must specify an input directory");
Contract.Requires(Directory.Exists(arguments.InputDirectory), "The input directory must exist");
var collectedArtifacts = new MultiValueDictionary <int, MLArtifact>();
var currentTicks = Environment.TickCount;
var linearFile = $"{Path.Combine(arguments.InputDirectory, $"{Convert.ToString(currentTicks)}.csv")}";
var linearOutput = TextWriter.Synchronized(new StreamWriter(linearFile));
s_logger.Info($"Linearizing to [{linearFile}]");
// write the headers
MLArtifact.WriteColumnsToStream(linearOutput);
// so now we are ready to linearize
var linearizeBlock = new TransformBlock <LinearizeArtifactsInput, TimedActionResult <LinearizeArtifactsOutput> >(i =>
{
var action = new LinearizeArtifacts(linearOutput);
return(action.PerformAction(i));
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.AppConfig.ConcurrencyConfig.MaxArtifactLinearizationTasks
}
);
var collectLinearResultsBlock = new ActionBlock <TimedActionResult <LinearizeArtifactsOutput> >(i =>
{
if (i.ExecutionStatus)
{
collectedArtifacts.Add(i.Result.NumQueues, i.Result.Linear);
}
},
// enforce serial
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
}
);
// connect
linearizeBlock.LinkTo(collectLinearResultsBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
// and post the tasks
var posted = 0;
foreach (var hashDir in Directory.EnumerateDirectories(arguments.InputDirectory))
{
linearizeBlock.Post(new LinearizeArtifactsInput(hashDir));
++posted;
}
s_logger.Info($"Posted {posted} linearizing tasks, waiting...");
linearizeBlock.Complete();
// and wait
collectLinearResultsBlock.Completion.Wait();
// and close...
linearOutput.Close();
// now, scale to create the samples...
s_logger.Info($"Creating {arguments.NumSamples} samples of size {arguments.SampleSize}");
var scale = new Dictionary <int, int>();
foreach (var entry in collectedArtifacts)
{
var queueCount = entry.Key;
var entryCount = entry.Value.Count;
var proportion = 1.0 * Math.BigMul(entryCount, arguments.SampleSize) / (1.0 * posted);
scale[queueCount] = (int)Math.Ceiling(proportion);
}
// we have the scale, lets post tasks here
var createSampleBlocks = new ActionBlock <SampleArtifactsInput>(i =>
{
var action = new SampleArtifacts();
action.PerformAction(i);
},
// one per each sample
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = arguments.NumSamples
}
);
// post some tasks in here
for (var i = 0; i < arguments.NumSamples; ++i)
{
createSampleBlocks.Post(new SampleArtifactsInput($"{Path.Combine(arguments.InputDirectory, $"{Convert.ToString(currentTicks)}-sample{i}.csv")}", scale, collectedArtifacts));
}
// and wait...
createSampleBlocks.Complete();
createSampleBlocks.Completion.Wait();
// done...
}
public static void Start(List <string> urls)
{
StringComparison comparison = StringComparison.InvariantCultureIgnoreCase;
Predicate <string> linkFilter = link =>
link.IndexOf(".aspx", comparison) != -1 ||
link.IndexOf(".php", comparison) != -1 ||
link.IndexOf(".htm", comparison) != -1 ||
link.IndexOf(".html", comparison) != -1 ||
link.EndsWith(".com", comparison) ||
link.EndsWith(".net", comparison);
Predicate <string> imgFilter = url =>
url.EndsWith(".jpg", comparison) ||
url.EndsWith(".png", comparison) ||
url.EndsWith(".gif", comparison);
var downloader = new TransformBlock <string, string>(
async(url) =>
{
// using IOCP the thread pool worker thread does return to the pool
var client = new HttpClient();
string result = await client.GetStringAsync(url);
return(result);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 2
});
var contentBroadcaster = new BroadcastBlock <string>(s => s);
var linkParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var links = _linkRegexHRef.Matches(html);
foreach (Match item in links)
{
var value = item.Groups["LINK"].Value;
output.Add(value);
}
return(output);
});
var imgParser = new TransformManyBlock <string, string>(
(html) =>
{
var output = new List <string>();
var images = _imgRegex.Matches(html);
foreach (Match item in images)
{
var value = item.Groups["IMG"].Value;
output.Add(value);
}
return(output);
});
var writer = new ActionBlock <string>(async url =>
{
var client = new HttpClient();
// using IOCP the thread pool worker thread does return to the pool
byte[] buffer = await client.GetByteArrayAsync(url);
string fileName = Path.GetFileName(url);
Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"\Images\");
string name = AppDomain.CurrentDomain.BaseDirectory + @"\Images\" + fileName;
using (Stream srm = File.OpenWrite(name))
{
await srm.WriteAsync(buffer, 0, buffer.Length);
}
});
var linkBroadcaster = new BroadcastBlock <string>(s => s);
IDisposable disposeAll = new CompositeDisposable(
// from [downloader] to [contentBroadcaster]
downloader.LinkTo(contentBroadcaster),
// from [contentBroadcaster] to [imgParser]
contentBroadcaster.LinkTo(imgParser),
// from [contentBroadcaster] to [linkParserHRef]
contentBroadcaster.LinkTo(linkParser),
// from [linkParser] to [linkBroadcaster]
linkParser.LinkTo(linkBroadcaster),
// conditional link to from [linkBroadcaster] to [downloader]
linkBroadcaster.LinkTo(downloader, linkFilter),
// from [linkBroadcaster] to [writer]
linkBroadcaster.LinkTo(writer, imgFilter),
// from [imgParser] to [writer]
imgParser.LinkTo(writer));
foreach (var url in urls)
{
downloader.Post(url);
}
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
downloader.Complete();
disposeAll.Dispose();
}
/// <summary>
/// Initializes a new instance of the <see cref="MultiplexerClientChannel"/> class.
/// </summary>
/// <param name="builder">The channel builder.</param>
/// <param name="count">The number of channels to create.</param>
/// <param name="inputBufferSize">The input buffer bounded capacity.</param>
/// <param name="outputBufferSize">The output buffer bounded capacity.</param>
/// <param name="channelCommandProcessor">The workflow for processing commands.</param>
/// <exception cref="System.ArgumentNullException"></exception>
/// <exception cref="System.ArgumentOutOfRangeException"></exception>
public MultiplexerClientChannel(
IEstablishedClientChannelBuilder builder,
int count = 5,
int inputBufferSize = 1,
int outputBufferSize = 1,
IChannelCommandProcessor channelCommandProcessor = null)
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
if (count <= 0)
{
throw new ArgumentOutOfRangeException(nameof(count));
}
// Create observable collections to allow synchronization
var channelCreatedHandlers = new ObservableCollection <Func <ChannelInformation, Task> >();
var channelDiscardedHandlers = new ObservableCollection <Func <ChannelInformation, Task> >();
var channelCreationFailedHandlers = new ObservableCollection <Func <FailedChannelInformation, Task <bool> > >();
var channelOperationFailedHandlers = new ObservableCollection <Func <FailedChannelInformation, Task <bool> > >();
ChannelCreatedHandlers = channelCreatedHandlers;
ChannelDiscardedHandlers = channelDiscardedHandlers;
ChannelCreationFailedHandlers = channelCreationFailedHandlers;
ChannelOperationFailedHandlers = channelOperationFailedHandlers;
// Global input buffers
var inputOptions = new ExecutionDataflowBlockOptions()
{
BoundedCapacity = inputBufferSize,
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
};
_inputMessageBufferBlock = new BufferBlock <Message>(inputOptions);
_inputNotificationBufferBlock = new BufferBlock <Notification>(inputOptions);
_channelCommandProcessor = channelCommandProcessor ?? new ChannelCommandProcessor();
// Uses the same channel command processor for all instances
// to avoid problems with commands responses being received on different channels.
builder.ChannelBuilder.WithChannelCommandProcessor(_channelCommandProcessor);
_processCommandTransformBlock = new TransformBlock <Command, Command>(c =>
{
if (_channelCommandProcessor.TrySubmitCommandResult(c))
{
return(null);
}
return(c);
},
inputOptions);
_inputCommandBufferBlock = new BufferBlock <Command>(inputOptions);
_processCommandTransformBlock.LinkTo(_inputCommandBufferBlock, c => c != null);
_processCommandTransformBlock.LinkTo(DataflowBlock.NullTarget <Command>(), c => c == null);
// The global output buffer
_outputBufferBlock = new BufferBlock <Envelope>(new DataflowBlockOptions()
{
BoundedCapacity = outputBufferSize
});
// An output action block per channel
_outputActionBlocks = new ActionBlock <Envelope> [count];
_channels = new IOnDemandClientChannel[count];
_listeners = new IChannelListener[count];
for (var i = 0; i < _channels.Length; i++)
{
// Add an instance suffix to the builder
var currentBuilder = builder
.Copy()
.WithInstance($"{builder.Instance}-{i+1}");
var channel = new OnDemandClientChannel(currentBuilder);
// Synchronize the handlers
AttachCollection(channelCreatedHandlers, channel.ChannelCreatedHandlers);
AttachCollection(channelDiscardedHandlers, channel.ChannelDiscardedHandlers);
AttachCollection(channelCreationFailedHandlers, channel.ChannelCreationFailedHandlers);
AttachCollection(channelOperationFailedHandlers, channel.ChannelOperationFailedHandlers);
// Setup the listener for the channel
_listeners[i] = new DataflowChannelListener(
_inputMessageBufferBlock,
_inputNotificationBufferBlock,
_processCommandTransformBlock);
// Create a single bounded action block for each channel
_outputActionBlocks[i] = new ActionBlock <Envelope>(async e => await SendToChannelAsync(channel, e).ConfigureAwait(false),
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1, MaxDegreeOfParallelism = 1
});
_outputBufferBlock.LinkTo(_outputActionBlocks[i], new DataflowLinkOptions()
{
PropagateCompletion = true
});
_channels[i] = channel;
}
_semaphore = new SemaphoreSlim(1, 1);
}
public async Task TestEncapsulate_EncapsulateBoundedTarget()
{
// source->||BoundedTransform->buffer||->sink
int messagesReceived = 0;
var transform = new TransformBlock<int, int>(x => {
messagesReceived++;
return x;
}, new ExecutionDataflowBlockOptions() { BoundedCapacity = 1 });
var buffer = new BufferBlock<int>();
transform.LinkTo(buffer);
var ignored = transform.Completion.ContinueWith(completion => buffer.Complete(), TaskScheduler.Default);
IPropagatorBlock<int, int> encapsulated = DataflowBlock.Encapsulate(transform, buffer);
encapsulated.LinkTo(new ActionBlock<int>(x => { }));
var source = new BufferBlock<int>();
source.LinkTo(encapsulated);
ignored = source.Completion.ContinueWith(completion => encapsulated.Complete(), TaskScheduler.Default);
// Feed
const int messagesSent = 10;
source.PostRange(0, messagesSent);
source.Complete();
await encapsulated.Completion;
Assert.Equal(messagesReceived, messagesSent);
}
public async Task DoNetwork()
{
OpenCL.IsEnabled = false;
var cacheFile = System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\stitch_cache.json";
State state = new State();
try
{
state = JsonConvert.DeserializeObject <State>(File.ReadAllText(cacheFile));
Console.WriteLine($"Using state from {cacheFile}");
} catch
{
Console.WriteLine("Couldn't load cache");
}
//state = new State();
AppState = state;
var workerPool = new LimitedConcurrencyLevelTaskScheduler(Math.Max(Environment.ProcessorCount - 2, 1)); // / 2, 1));
//var workerPool = new LimitedConcurrencyLevelTaskScheduler(1);
var snapshotState = new ActionBlock <State>((s) =>
{
var content = "";
s.Lock(lockedState => content = JsonConvert.SerializeObject(lockedState));
WriteAllTextWithBackup(cacheFile, content);
this.Dispatcher.Invoke(() => Joins.ItemsSource = AppState.Joins);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
state.ChangeListener = snapshotState;
var blockOptions = new ExecutionDataflowBlockOptions
{
TaskScheduler = workerPool,
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded // Handled by underlying scheduler
};
//var sourceDir = "C:/Users/Xavier/Source/ori-tracker/MapStitcher/Screenshots";
var sourceDir = "C:/Users/Xavier/Source/ori-tracker/MapStitcher/4KScreenshots";
/*
* IMagickImage image1 = new MagickImage(System.IO.Path.GetFullPath($"{sourceDir}/../Temp/forlorn-1.png"));
* image1 = image1.Clone();
* var settings = new MorphologySettings
* {
* Channels = Channels.Alpha,
* Method = MorphologyMethod.Distance,
* Kernel = Kernel.Euclidean,
* KernelArguments = "1,50!"
* };
*
* image1.Alpha(AlphaOption.Set);
* image1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
* image1.Morphology(settings);
* image1.Write(System.IO.Path.GetFullPath($"{sourceDir}/../Temp/forlorn-test.png"));
*/
/*
* MagickImage image1 = new MagickImage($"{sourceDir}/sorrow-1.png");
* MagickImage image2 = new MagickImage($"{sourceDir}/sorrow-2.png");
*/
/*
* var sourceFiles = new List<string>
* {
* $"{sourceDir}\\387290_20180314160604_1.png",
* };
*/
var sourceFiles = Directory.GetFiles(sourceDir, "*.png");
/*
* var sourceFiles = new List<string>
* {
* $"{sourceDir}\\forlorn-1.png",
* $"{sourceDir}\\forlorn-2.png",
* $"{sourceDir}\\forlorn-3.png",
* };
*/
//state.ClearJoins();
/*
* state.ClearNeedle(new NeedleKey { Key = $"{sourceDir}/forlorn-3.png", Gravity = Gravity.West });
* state.ClearJoin($"{sourceDir}/forlorn-2.png", $"{sourceDir}/forlorn-3.png");
* state.ClearJoin($"{sourceDir}/forlorn-1.png", $"{sourceDir}/forlorn-3.png");
* state.ClearJoin($"{sourceDir}/forlorn-2.png", $"{sourceDir}/forlorn-1.png");
*/
this.Dispatcher.Invoke(() => SourceImages.ItemsSource = SourceImages2.ItemsSource = sourceFiles);
this.Dispatcher.Invoke(() => Joins.ItemsSource = AppState.Joins);
UpdateUI();
var loadFromDiskBlock = new TransformBlock <string, string>(path =>
{
// TODO: Make this a load and crop task
var task = new StitchTask($"Load and crop {System.IO.Path.GetFileName(path)}");
this.Dispatcher.Invoke(() => Tasks.Add(task));
state.GetOrAddImage(path, () =>
{
var image = new MagickImage(path);
var originalSize = new Size(image.Width, image.Height);
int sideMargin = (int)(image.Width * 0.15); // The sides have a subtle animated mask over them. 280px wide on 1920px resolution. Crop them out.
int topMargin = (int)(image.Height * 0.17);
int bottomMargin = (int)(image.Height * 0.15);
var bounds = new MagickGeometry(sideMargin, topMargin, image.Width - sideMargin * 2, image.Height - bottomMargin - topMargin);
image.Crop(bounds);
image.RePage();
//image.Write("C:\\Users\\Xavier\\Source\\ori-tracker\\MapStitcher\\Temp\\" + System.IO.Path.GetFileName(path));
return(image);
});
task.Complete("Done", false);
return(path);
}, blockOptions);
var gravities = new TransformManyBlock <string, NeedleKey>(path =>
{
return(allGravities.Select(g => new NeedleKey()
{
Key = path, Gravity = g
}));
}, blockOptions);
var findNeedleBlock = new TransformBlock <NeedleKey, NeedleKey>(needle =>
{
var task = new FindNeedleTask(state, needle);
this.Dispatcher.Invoke(() => Tasks.Add(task));
task.Run();
return(needle);
}, blockOptions);
var findJoinBlock = new TransformBlock <SearchKey, string>(t =>
{
var haystack = t.Item1;
var needle = t.Item2;
var task = new SearchTask(state, haystack, needle);
this.Dispatcher.Invoke(() => Tasks.Add(task));
task.Run();
completedSearchTasks++;
return(haystack); // TODO: Figure out best thing to propagate. Maybe when match found?
}, blockOptions);
var broadcaster = new BroadcastBlock <string>(null);
var cartesian = new CartesianProductBlock <string, NeedleKey>();
var propagate = new DataflowLinkOptions {
PropagateCompletion = true
};
var headBlock = loadFromDiskBlock;
headBlock.LinkTo(broadcaster, propagate);
broadcaster.LinkTo(gravities, propagate);
gravities.LinkTo(findNeedleBlock, propagate);
// Don't propagate completion from left/right sources for cartesian join. It should
// complete when _both_ are done (which is it's default behaviour)
broadcaster.LinkTo(cartesian.Left, propagate);
findNeedleBlock.LinkTo(cartesian.Right, propagate);
var countTotals = new TransformManyBlock <Tuple <string, NeedleKey>, SearchKey>(t =>
{
var haystack = t.Item1;
var needle = t.Item2;
var none = Enumerable.Empty <SearchKey>();
if (haystack == needle.Key || !state.GetNeedle(needle).MeetsThreshold())
{
return(none);
}
var existingJoins = state.Joins;
var connectedJoins = new HashSet <HashSet <string> >();
foreach (var join in existingJoins)
{
var found = false;
foreach (var connectedSubset in connectedJoins)
{
if (connectedSubset.Contains(join.Image1) || connectedSubset.Contains(join.Image2))
{
connectedSubset.Add(join.Image1);
connectedSubset.Add(join.Image2);
found = true;
break;
}
}
if (!found)
{
var newSubset = new HashSet <string>();
newSubset.Add(join.Image1);
newSubset.Add(join.Image2);
connectedJoins.Add(newSubset);
}
}
connectedJoins.Aggregate(new HashSet <HashSet <string> >(), (acc, x) => {
var found = false;
foreach (var connectedSubset in acc)
{
if (connectedSubset.Overlaps(x))
{
connectedSubset.UnionWith(x);
found = true;
break;
}
}
if (!found)
{
acc.Add(x);
}
return(acc);
});
if (connectedJoins.Any(x => x.Contains(haystack) && x.Contains(needle.Key)))
{
Console.WriteLine("Two images already connected via transitive joins, skipping");
return(none);
}
totalSearchTasks++;
return(Enumerable.Repeat(SearchKey.Create(t.Item1, t.Item2), 1));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
cartesian.LinkTo(countTotals, propagate);
countTotals.LinkTo(findJoinBlock, propagate);
var sink = new ActionBlock <string>(s => { });
findJoinBlock.LinkTo(sink, propagate);
foreach (var file in sourceFiles)
{
headBlock.Post(file);
}
headBlock.Complete();
await sink.Completion.ContinueWith(_ =>
{
Console.WriteLine("Pipeline Finished");
/*
* this.Dispatcher.Invoke(() => TaskGrid.ItemsSource = Tasks.Where(x =>
* {
* if (x is SearchTask)
* {
* var task = (SearchTask)x;
* return task.Name.Contains("Found");
* return task.searchResult.MeetsThreshold();
* return task.searchResult.Distance < 2500;
* }
* return false;
* }));
*/
Dictionary <string, Point> completedJoins = new Dictionary <string, Point>();
var remainingJoins = new List <State.Join>(state.Joins);
var rejects = new List <State.Join>();
var images = new MagickImageCollection();
var lastCycleCount = 0;
var morphologySettings = new MorphologySettings
{
Channels = Channels.Alpha,
Method = MorphologyMethod.Distance,
Kernel = Kernel.Euclidean,
KernelArguments = "1,50!"
};
while (remainingJoins.Count > 0 && remainingJoins.Count != lastCycleCount)
{
lastCycleCount = remainingJoins.Count;
foreach (var join in remainingJoins)
{
if (completedJoins.Count == 0)
{
var tempPath = System.IO.Path.GetTempFileName();
var tempPath2 = System.IO.Path.GetTempFileName();
// Initial seed
var i1 = state.Image(join.Image1).Clone();
var i2 = state.Image(join.Image2).Clone();
i1.Alpha(AlphaOption.Set);
i1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i1.Morphology(morphologySettings);
i1.Write(tempPath);
i1.Dispose();
i1 = new MagickImage(tempPath);
i1.BackgroundColor = new MagickColor(18, 18, 18);
i2.Alpha(AlphaOption.Set);
i2.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i2.Morphology(morphologySettings);
i2.Write(tempPath2);
i2.Dispose();
i2 = new MagickImage(tempPath2);
i2.Page = new MagickGeometry($"{ToOffset(join.JoinPoint.X)}{ToOffset(join.JoinPoint.Y)}");
images.Add(i1);
images.Add(i2);
completedJoins.Add(join.Image1, new Point(0, 0));
completedJoins.Add(join.Image2, join.JoinPoint);
File.Delete(tempPath);
File.Delete(tempPath2);
}
else
{
Point offset = join.JoinPoint;
if (completedJoins.ContainsKey(join.Image1) && completedJoins.ContainsKey(join.Image2))
{
// NOOP
}
else if (completedJoins.ContainsKey(join.Image1))
{
completedJoins.TryGetValue(join.Image1, out offset);
var tempPath = System.IO.Path.GetTempFileName();
var i2 = state.Image(join.Image2).Clone();
var joinPoint = new Point(join.JoinPoint.X + offset.X, join.JoinPoint.Y + offset.Y);
i2.Alpha(AlphaOption.Set);
i2.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i2.Morphology(morphologySettings);
i2.Write(tempPath);
//i2.Dispose();
//i2 = new MagickImage(tempPath);
i2.Page = new MagickGeometry($"{ToOffset(joinPoint.X)}{ToOffset(joinPoint.Y)}");
images.Add(i2);
File.Delete(tempPath);
completedJoins.Add(join.Image2, joinPoint);
}
else if (completedJoins.ContainsKey(join.Image2))
{
completedJoins.TryGetValue(join.Image2, out offset);
var tempPath = System.IO.Path.GetTempFileName();
var i1 = state.Image(join.Image1).Clone();
var joinPoint = new Point(offset.X - join.JoinPoint.X, offset.Y - join.JoinPoint.Y);
i1.Alpha(AlphaOption.Set);
i1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i1.Morphology(morphologySettings);
i1.Write(tempPath);
//i1.Dispose();
//i1 = new MagickImage(tempPath);
i1.Page = new MagickGeometry($"{ToOffset(joinPoint.X)}{ToOffset(joinPoint.Y)}");
images.Add(i1);
File.Delete(tempPath);
completedJoins.Add(join.Image1, joinPoint);
}
else
{
rejects.Add(join);
}
}
}
remainingJoins = rejects.ToList();
rejects.Clear();
}
if (images.Any())
{
var merged = images.Merge();
//merged.BackgroundColor = new MagickColor(0, 0, 0);
//merged.Alpha(AlphaOption.Remove);
//merged.Alpha(AlphaOption.Off);
merged.Write("C:\\Users\\Xavier\\Source\\ori-tracker\\MapStitcher\\Temp\\map.png");
DisplayImage(Viewer, merged);
Console.WriteLine("Done Compositing");
}
});
}
