public void Run()
{
this.output.WriteLine("TPL start");
this.CheckFileExists();
var f = new FileInfo(this.sampleLogFileName);
this.fileSizeInBytes = f.Length;
this.lineSizeInBytesSoFar = 0;
this.lineCount = 0;
var sw = new Stopwatch();
sw.Start();
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var ab = new ActionBlock<string>(s => this.ProcessLine(s), options);
using (TextReader reader = File.OpenText(this.sampleLogFileName))
{
string line;
while ((line = reader.ReadLine()) != null)
{
ab.Post(line);
}
}
ab.Complete();
ab.Completion.Wait();
sw.Stop();
this.ShowResults();
this.output.WriteLine();
this.output.WriteLine("TPL done in {0}", sw.Elapsed);
}
public void BasicUsageTest ()
{
bool act1 = false, act2 = false;
var evt = new CountdownEvent (2);
var broadcast = new BroadcastBlock<int> (null);
var action1 = new ActionBlock<int> (i =>
{
act1 = i == 42;
evt.Signal ();
});
var action2 = new ActionBlock<int> (i =>
{
act2 = i == 42;
evt.Signal ();
});
broadcast.LinkTo (action1);
broadcast.LinkTo (action2);
Assert.IsTrue (broadcast.Post (42));
Assert.IsTrue (evt.Wait (100));
Assert.IsTrue (act1);
Assert.IsTrue (act2);
}
public async Task TestDataBroadcaster1()
{
var random = new Random();
var dataCopier = new DataBroadcaster<int>();
int sum1 = 0;
int sum2 = 0;
var action1 = new ActionBlock<int>(i => sum1 = sum1 + i);
var action2 = new ActionBlock<int>(i => sum2 = sum2 + i);
dataCopier.LinkTo(DataflowUtils.FromBlock(action1));
dataCopier.LinkTo(DataflowUtils.FromBlock(action2));
for (int j = 0; j < 1000; j++)
{
dataCopier.InputBlock.Post((int) (random.NextDouble()*10000));
}
dataCopier.Complete();
await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion);
Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2);
Assert.AreEqual(sum1, sum2);
}
public Engine(IWorkItemRepository repository, IActivityRunner activityRunner,
IStateMachineProvider stateMachineProvider)
{
if (repository == null) throw new ArgumentNullException("repository");
if (activityRunner == null) throw new ArgumentNullException("activityRunner");
if (stateMachineProvider == null) throw new ArgumentNullException("stateMachineProvider");
_repository = repository;
_activityRunner = activityRunner;
_stateMachineProvider = stateMachineProvider;
_stateQueue = new ActionBlock<int>(id => UpdateState(id),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
_workerQueue = new ActionBlock<int>(id => RunActivity(id),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = int.MaxValue
});
_stateQueue.Completion.ContinueWith(t => { _workerQueue.Complete(); }, TaskContinuationOptions.OnlyOnFaulted);
_workerQueue.Completion.ContinueWith(t => { ((IDataflowBlock) _stateQueue).Fault(t.Exception); },
TaskContinuationOptions.OnlyOnFaulted);
}
public async Task TestDataBroadcaster2()
{
var random = new Random();
var buffer = new BufferBlock<int>();
int sum1 = 0;
int sum2 = 0;
int sum3 = 0;
var action1 = new ActionBlock<int>(i => sum1 = sum1 + i);
var action2 = new ActionBlock<int>(i => sum2 = sum2 + i);
var action3 = new ActionBlock<int>(i => sum3 = sum3 + i);
buffer.ToDataflow().LinkToMultiple(new []{action1, action2, action3}.Select(a => a.ToDataflow()).ToArray());
for (int j = 0; j < 1000; j++)
{
buffer.Post((int)(random.NextDouble() * 10000));
}
buffer.Complete();
await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion, action3.Completion);
Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2);
Assert.AreEqual(sum1, sum2);
Assert.AreEqual(sum1, sum3);
}
public void Consumer(ISourceBlock<string> source)
{
var ablock = new ActionBlock<string>(
data => this.ParseRawIRC(data), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
source.LinkTo(ablock);
}
public EventMapper(BleSystemConfiguration systemConfiguration)
{
var receiverPaths = systemConfiguration.ReceiverPaths;
_useWeightedPath = systemConfiguration.UseWeightedPaths;
_receiverPaths = receiverPaths.ToDictionary(rp => rp);
_eventGroupHandler = new ActionBlock<List<SignalEventDetails>>(deg => HandleEventGroup(deg), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
_eventsByReceiver = new Dictionary<BleReceiver, Dictionary<string, SignalEventDetails>>(systemConfiguration.BleReceivers.Count);
_monitoredReceivers = systemConfiguration.BleReceivers.Values.ToList();
_monitoredTransmitters = systemConfiguration.BleTransmitters.Values.ToList();
_receiverPathsTree = new Dictionary<BleReceiver, Dictionary<ReceiverPath, ReceiverPath>>();
foreach (var bleReceiver in _monitoredReceivers)
{
if (_eventsByReceiver.ContainsKey(bleReceiver))
continue;
var pathDictionary = new Dictionary<ReceiverPath, ReceiverPath>();
_receiverPathsTree.Add(bleReceiver, pathDictionary);
foreach (var receiverPath in receiverPaths)
{
if (Equals(receiverPath.From, bleReceiver) || Equals(receiverPath.To, bleReceiver))
pathDictionary.Add(receiverPath, receiverPath);
}
var receiverTransmitters = new Dictionary<string, SignalEventDetails>(_monitoredTransmitters.Count);
_monitoredTransmitters.ForEach(t =>
{
if (receiverTransmitters.ContainsKey(t.MacAddress))
return;
receiverTransmitters.Add(t.MacAddress,
new SignalEventDetails { Transmitter = t, BleReceiver = bleReceiver });
});
_eventsByReceiver.Add(bleReceiver, receiverTransmitters);
}
_scanTimer = new Timer(ScanInterval);
_scanTimer.Elapsed += ScanIntervalElapsed;
_scanTimer.Enabled = true;
}
public void TriggerBatchTest ()
{
int[] array = null;
var evt = new ManualResetEventSlim (false);
var buffer = new BatchBlock<int> (10);
var block = new ActionBlock<int[]> (i =>
{
array = i;
evt.Set ();
});
buffer.LinkTo (block);
for (int i = 0; i < 9; i++)
Assert.IsTrue (buffer.Post (i));
buffer.TriggerBatch ();
evt.Wait ();
Assert.IsNotNull (array);
Assert.IsTrue (buffer.Post (42));
evt.Wait (1600);
CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 },
array);
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Signaling completion to the source block.");
bufferBlock.Complete();
Console.WriteLine("Done.");
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(200);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(500);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
Console.WriteLine("Waiting for the completion to be propagated through the network...");
actionBlock.Completion.ContinueWith(t =>
{
Console.WriteLine("Finished processing.");
Console.WriteLine(string.Format("Completion status: {0}.", t.Status));
}).Wait();
}
private FileLogWriter(string filePath) {
_filePath = filePath;
_messages = new ActionBlock<string>(new Func<string, Task>(WriteToFile));
AppDomain.CurrentDomain.ProcessExit += OnProcessExit;
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
}
public void BasicUsageTest ()
{
int[] array = null;
var evt = new ManualResetEventSlim (false);
var buffer = new BatchBlock<int> (10);
var block = new ActionBlock<int[]> (i =>
{
array = i;
evt.Set ();
});
buffer.LinkTo<int[]> (block);
for (int i = 0; i < 9; i++)
Assert.IsTrue (buffer.Post (i));
Assert.IsFalse (evt.Wait (100));
Assert.IsNull (array);
Assert.IsTrue (buffer.Post (42));
Assert.IsTrue (evt.Wait (1000));
Assert.IsNotNull (array);
CollectionAssert.AreEqual (new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 42 }, array);
}
public void ProduceLogs(int count, int buffSize)
{
var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize };
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.buff.log", false);
BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6);
writer.SendAsync(line).Wait();
}
buffer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
static Publisher()
{
_trackerBlock = new ActionBlock<ContainerBase>(async tc =>
{
try
{
ContainerBase trackerContainer = tc;
if (_firstTC == null)
_firstTC = trackerContainer;
else if (_firstTC == trackerContainer &&
!_trackerBlockCancellationTokenSource.IsCancellationRequested)
{
try
{
await
Task.Delay(TimeSpan.FromSeconds(15), _trackerBlockCancellationTokenSource.Token)
.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
Configurator.Configuration.Logger.Info("Task.Delay exiting");
}
}
foreach (
TrackerData td in
trackerContainer.GetTrackerData(_trackerBlockCancellationTokenSource.IsCancellationRequested)
)
{
_publisherBlock.Post(td);
}
}
catch (Exception ex)
{
Configurator.Configuration.Logger.Error(ex.Message, ex);
}
finally
{
if (!_trackerBlockCancellationTokenSource.IsCancellationRequested)
_trackerBlock.Post(tc);
else if (_trackerBlock.InputCount == 0)
_trackerBlock.Complete();
}
});
_publisherBlock = new ActionBlock<TrackerData>(tc =>
{
try
{
_lastPersistanceComplete = false;
Configurator.Configuration.Persister.Persist(tc);
_lastPersistanceComplete = true;
}
catch (Exception ex)
{
Configurator.Configuration.Logger.Error(ex.Message, ex);
_lastPersistanceComplete = true;
}
}, new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 4});
}
public MatchPipeline(RiotApi riotApi, RiotQuerySettings querySettings, IMatchDetailProcessor matchDetailProcessor)
{
api = riotApi;
this.querySettings = querySettings;
this.matchDetailProcessor = matchDetailProcessor;
queryQueues.Add(querySettings.Queue);
// Create match producer
if (!querySettings.NoDownload)
{
playerMatchProducer = new PlayerMatchProducer(api, querySettings, queryQueues, testSynchronizer);
// Create blocks
ConsumeMatchBlock = new TransformBlock<MatchSummary, MatchDetail>(
async match => await ConsumeMatch(match),
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 5 });
}
ConsumeMatchDetailBlock = new ActionBlock<MatchDetail>(
async match => await matchDetailProcessor.ConsumeMatchDetail(match),
new ExecutionDataflowBlockOptions() {
MaxDegreeOfParallelism = matchDetailProcessor.MaxDegreeOfParallelism
});
// Link blocks
if (!querySettings.NoDownload)
{
playerMatchProducer.MatchProducerBlock.LinkTo(ConsumeMatchBlock, new DataflowLinkOptions() { PropagateCompletion = false });
ConsumeMatchBlock.LinkTo(ConsumeMatchDetailBlock, new DataflowLinkOptions() { PropagateCompletion = false }, match => match != null);
ConsumeMatchBlock.LinkTo(DataflowBlock.NullTarget<MatchDetail>(), new DataflowLinkOptions() { PropagateCompletion = false });
}
}
/// <summary>
/// Initialize a new instance of the <see cref="DataflowWebHookSender"/> with the given retry policy, <paramref name="options"/>,
/// and <paramref name="httpClient"/>. This constructor is intended for unit testing purposes.
/// </summary>
internal DataflowWebHookSender(
ILogger logger,
IEnumerable<TimeSpan> retryDelays,
ExecutionDataflowBlockOptions options,
HttpClient httpClient,
Action<WebHookWorkItem> onWebHookSuccess,
Action<WebHookWorkItem> onWebHookFailure)
: base(logger)
{
retryDelays = retryDelays ?? DefaultRetries;
options = options ?? new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = DefaultMaxConcurrencyLevel
};
_httpClient = httpClient ?? new HttpClient();
// Create the launch processors with the given retry delays
_launchers = new ActionBlock<WebHookWorkItem>[1 + retryDelays.Count()];
int offset = 0;
_launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await LaunchWebHook(item), options);
foreach (TimeSpan delay in retryDelays)
{
_launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await DelayedLaunchWebHook(item, delay));
}
string msg = string.Format(CultureInfo.CurrentCulture, CustomResources.Manager_Started, typeof(DataflowWebHookSender).Name, _launchers.Length);
Logger.Info(msg);
// Set handlers for testing purposes
_onWebHookSuccess = onWebHookSuccess;
_onWebHookFailure = onWebHookFailure;
}
public void TestActionBlockConstructor()
{
// SYNC
// without option
var block = new ActionBlock<int>(i => { });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock." + block.InputCount);
//with not cancelled token and default scheduler
block = new ActionBlock<int>(i => { }, new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new ActionBlock<int>(i => { }, new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock.");
// ASYNC (a copy of the sync but with constructors returning Task instead of void
var dummyTask = new Task(() => { });
// without option
block = new ActionBlock<int>(i => dummyTask);
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock.");
//with not cancelled token and default scheduler
block = new ActionBlock<int>(i => dummyTask, new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock.");
//with a cancelled token and default scheduler
token = new CancellationToken(true);
block = new ActionBlock<int>(i => dummyTask, new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token });
Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new ActionBlock.");
}
public RobotStateDashboard()
{
dataBlockState = new ActionBlock<IRobotState>(v => { Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { StateDataText = v.ToString(); }).AsTask().Wait(); });
dataBlockPose = new ActionBlock<IRobotPose>(v => { Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { PoseDataText = v.ToString(); }).AsTask().Wait(); });
this.InitializeComponent();
}
public void BasicUsageTest ()
{
Tuple<int, int> tuple = null;
var evt = new ManualResetEventSlim (false);
var ablock = new ActionBlock<Tuple<int, int>> (t =>
{
tuple = t;
evt.Set ();
});
var block = new JoinBlock<int, int> ();
block.LinkTo (ablock);
block.Target1.Post (42);
evt.Wait (1000);
Assert.IsNull (tuple);
block.Target2.Post (24);
evt.Wait ();
Assert.IsNotNull (tuple);
Assert.AreEqual (42, tuple.Item1);
Assert.AreEqual (24, tuple.Item2);
}
public void Start()
{
var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink);
var source = new BufferBlock<GetPageMessage>();
var linkOptions = new DataflowLinkOptions {PropagateCompletion = false};
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var worker = new TransformBlock<GetPageMessage, PageResultMessage>(
(Func<GetPageMessage, PageResultMessage>)Worker, options);
source.LinkTo(worker, linkOptions);
worker.LinkTo(sink, linkOptions);
}
foreach (var url in UrlList.Urls)
{
source.Post(new GetPageMessage{ Url = url });
}
source.Complete();
sink.Completion.Wait();
}
public void BasicUsageTest ()
{
Tuple<IList<int>, IList<int>, IList<string>> result = null;
var evt = new ManualResetEventSlim (false);
var actionBlock =
new ActionBlock<Tuple<IList<int>, IList<int>, IList<string>>> (r =>
{
result = r;
evt.Set ();
});
var block = new BatchedJoinBlock<int, int, string> (3);
block.LinkTo (actionBlock);
// all targets once
Assert.IsTrue (block.Target1.Post (1));
Assert.IsTrue (block.Target2.Post (2));
Assert.IsFalse (evt.Wait (100));
Assert.IsNull (result);
Assert.IsTrue (block.Target3.Post ("foo"));
Assert.IsTrue (evt.Wait (1000));
Assert.IsNotNull (result);
CollectionAssert.AreEqual (new[] { 1 }, result.Item1);
CollectionAssert.AreEqual (new[] { 2 }, result.Item2);
CollectionAssert.AreEqual (new[] { "foo" }, result.Item3);
}
public void MultipleBindingTest ()
{
BufferBlock<int> buffer = new BufferBlock<int> ();
var evt = new CountdownEvent (10);
int count = 0;
ActionBlock<int> block = new ActionBlock<int> ((i) => { Interlocked.Decrement (ref count); evt.Signal (); });
IDisposable bridge = buffer.LinkTo (block);
for (int i = 0; i < 10; i++)
Assert.IsTrue (buffer.Post (i));
evt.Wait ();
Assert.AreEqual (-10, count);
count = 0;
evt.Reset ();
bridge.Dispose ();
ActionBlock<int> block2 = new ActionBlock<int> ((i) => { Interlocked.Increment (ref count); evt.Signal (); });
buffer.LinkTo (block2);
for (int i = 0; i < 10; i++)
Assert.IsTrue (buffer.Post (i));
evt.Wait ();
Assert.AreEqual (10, count);
}
public ExecutionPipeline(Kernel kernel)
{
_kernel = kernel;
_commandQueue = new BufferBlock<CommandRequest[]>();
_queryQueue = new BatchBlock<QueryRequest>(MaxConcurrentQueries);
var transactionHandler = new ActionBlock<object>(t =>
{
if (t is QueryRequest[])
{
var queries = t as QueryRequest[];
Task[] tasks = queries.Select(q => Task.Factory.StartNew(_ => ExecuteQuery(q), null)).ToArray();
Task.WaitAll(tasks);
}
else if (t is CommandRequest[])
{
var commands = t as CommandRequest[];
foreach (var commandContext in commands)
{
var result = _kernel.Execute(commandContext.Command);
commandContext.Response.Post(result);
}
}
});
_commandQueue.LinkTo(transactionHandler);
_queryQueue.LinkTo(transactionHandler);
_timer = new Timer(_ => _queryQueue.TriggerBatch());
_timer.Change(Interval, Interval);
}
private void GetLastEventProcessedForHandlers()
{
//TODO calculate the lowest numbered LEP of all the handlers and use that for the start position
// ask each registered handler to get there last processed event and hold on to it.
var actionBlock = new ActionBlock<IHandler>(x => x.GetLastPositionProcessed(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 });
_eventHandlers.ForEach(async x=> await actionBlock.SendAsync(x));
}
public void Run()
{
var cts = new CancellationTokenSource();
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>(new DataflowBlockOptions { CancellationToken = cts.Token });
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Scheduling cancellation after 5 seconds.");
cts.CancelAfter(TimeSpan.FromSeconds(5));
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1, CancellationToken = cts.Token });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10, CancellationToken = cts.Token });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
var t1 = bufferBlock.Completion.ContinueWith(t => Console.WriteLine("Buffer block status: {0}", t.Status));
var t2 = actionBlock.Completion.ContinueWith(t => Console.WriteLine("Action block status: {0}", t.Status));
Console.WriteLine("Waiting for the network to finish.");
Task.WaitAll(t1, t2);
}
public void WriteOneLatency_OneMetricFlushed_Success()
{
_block = TimedLatencyAggregatorBlockFactory.CreateBlock(_outputBuffer,
String.Empty,
_intervalService,
true,
_log.Object);
var pulseDate = DateTime.Now;
_block.Post(new Timing("foo.bar.baz", 100));
_block.WaitUntilAllItemsProcessed();
_intervalService.Pulse(pulseDate);
Assert.AreEqual(new GraphiteLine("foo.bar.baz.count", 1, pulseDate.ToEpoch()),_outputBuffer.GetGraphiteLine(0));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.min", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(1));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.max", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(2));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.mean", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(3));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.sum", 100, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(4));
Assert.AreEqual(new GraphiteLine("foo.bar.baz.sumSquares", 10000, pulseDate.ToEpoch()), _outputBuffer.GetGraphiteLine(5));
Assert.AreEqual(6, _outputBuffer.GraphiteLines.Count);
// Ensure that min, max, mean and sum are all equal
Assert.IsTrue(_outputBuffer.GetGraphiteLine(1).Quantity == _outputBuffer.GetGraphiteLine(2).Quantity
&& _outputBuffer.GetGraphiteLine(2).Quantity == _outputBuffer.GetGraphiteLine(3).Quantity
&& _outputBuffer.GetGraphiteLine(3).Quantity == _outputBuffer.GetGraphiteLine(4).Quantity);
}
public static ITargetBlock<string> SetupPipeline()
{
var fileNamesForPath = new TransformBlock<string, IEnumerable<string>>(
path =>
{
return GetFileNames(path);
});
var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
fileNames =>
{
return LoadLines(fileNames);
});
var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(
lines2 =>
{
return GetWords(lines2);
});
var display = new ActionBlock<IEnumerable<string>>(
coll =>
{
foreach (var s in coll)
{
WriteLine(s);
}
});
fileNamesForPath.LinkTo(lines);
lines.LinkTo(words);
words.LinkTo(display);
return fileNamesForPath;
}
public void BasicUsageTest ()
{
bool act1 = false, act2 = false;
var evt = new CountdownEvent (2);
var block = new WriteOnceBlock<int> (null);
var action1 = new ActionBlock<int> (i =>
{
act1 = i == 42;
evt.Signal ();
});
var action2 = new ActionBlock<int> (i =>
{
act2 = i == 42;
evt.Signal ();
});
block.LinkTo (action1);
block.LinkTo (action2);
Assert.IsTrue (block.Post (42), "#1");
Assert.IsFalse (block.Post (43), "#2");
Assert.IsTrue (evt.Wait (1000), "#3");
Assert.IsTrue (act1, "#4");
Assert.IsTrue (act2, "#5");
}
public static IPropagatorBlock<File, FileLine> GetFileLinesEnumeratorBlock()
{
var resultsBlock = new BufferBlock<FileLine>();
var actionBlock = new ActionBlock<File>(
async file =>
{
using (var reader = new System.IO.StreamReader(new System.IO.FileStream(
file.FullPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.Read,
bufferSize: 4096,
useAsync: true)))
{
string line;
var row = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
resultsBlock.Post(new FileLine(file, row, line));
}
row++;
}
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
_waitHandle.Signal();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock);
transformBlock.LinkTo(actionBlock);
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
_waitHandle.Wait();
}
public CrawlScheduler(IUrlHasher urlHasher, IConfiguration configuration, ISinglePageCrawler pageCrawler, ICrawlUrlRepository crawlUrlRepository)
{
this.urlHasher = urlHasher;
this.configuration = configuration;
this.pageCrawler = pageCrawler;
this.crawlUrlRepository = crawlUrlRepository;
schedulingBlock = new ActionBlock<PageCrawlResult>(result =>
{
var websiteDefinition = result.CrawlUrl.WebsiteDefinition;
Interlocked.Increment(ref websiteDefinition.ProcessedUrlsCount);
RaisePageCrawled(result);
if (result.Links.Any())
{
Schedule(result.Links);
}
if (websiteDefinition.UrlsToProcessCount == websiteDefinition.ProcessedUrlsCount)
{
websiteProcessingDefinitions[websiteDefinition].Complete();
}
ScheduleNext();
});
}
protected virtual RemoteTcpPeer CreateRemoteTcpPeer(TcpClient tcpClient, SslStream sslStream, ActionBlock <RemoteTcpPeerOutgoingMessage> sendQueue, CancellationTokenSource tokenSource)
{
return(new RemoteTcpPeer(
this.Config.ProtocolFrameDefragmenterFactory,
tcpClient,
sslStream,
sendQueue,
tokenSource));
}
public void TestEtw()
{
RemoteInvoke(() =>
{
using (var listener = new TestEventListener(new Guid("16F53577-E41D-43D4-B47E-C17025BF4025"), EventLevel.Verbose))
{
ActionBlock <int> ab = null;
BufferBlock <int> bb = null;
int remaining = 0;
CountdownEvent ce = new CountdownEvent(0);
// Check that block creation events fire
const int DataflowBlockCreatedId = 1;
remaining = 2;
listener.RunWithCallback(ev => {
Assert.Equal(expected: DataflowBlockCreatedId, actual: ev.EventId);
remaining--;
},
() => {
ab = new ActionBlock <int>(i => { });
bb = new BufferBlock <int>(); // trigger block creation event
Assert.Equal(expected: 0, actual: remaining);
});
// Check that linking events fire
const int BlockLinkedId = 4;
remaining = 1;
IDisposable link = null;
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockLinkedId, actual: ev.EventId);
remaining--;
},
() => {
link = bb.LinkTo(ab);
Assert.Equal(expected: 0, actual: remaining);
});
// Check that unlinking events fire
const int BlockUnlinkedId = 5;
remaining = 1;
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockUnlinkedId, actual: ev.EventId);
remaining--;
},
() => {
link.Dispose();
Assert.Equal(expected: 0, actual: remaining);
});
// Check that task launched events fire
const int TaskLaunchedId = 2;
ce.Reset(1);
listener.RunWithCallback(ev => {
Assert.Equal(expected: TaskLaunchedId, actual: ev.EventId);
ce.Signal();
},
() => {
ab.Post(42);
ce.Wait();
Assert.Equal(expected: 0, actual: ce.CurrentCount);
});
// Check that completion events fire
const int BlockCompletedId = 3;
ce.Reset(2);
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockCompletedId, actual: ev.EventId);
ce.Signal();
},
() => {
ab.Complete();
bb.Complete();
ce.Wait();
Assert.Equal(expected: 0, actual: ce.CurrentCount);
});
}
return(SuccessExitCode);
}).Dispose();
}
static void Main()
{
// <snippet3>
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock <string, string>(async uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return(await new HttpClient().GetStringAsync(uri));
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character.
char[] tokens = text.Select(c => char.IsLetter(c) ? c : ' ').ToArray();
text = new string(tokens);
// Separate the text into an array of words.
return(text.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries));
});
// Removes short words and duplicates.
var filterWordList = new TransformBlock <string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return(words
.Where(word => word.Length > 3)
.Distinct()
.ToArray());
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock <string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
var wordsSet = new HashSet <string>(words);
return(from word in words.AsParallel()
let reverse = new string(word.Reverse().ToArray())
where word != reverse && wordsSet.Contains(reverse)
select word);
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock <string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
// </snippet3>
// <snippet4>
//
// Connect the dataflow blocks to form a pipeline.
//
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
downloadString.LinkTo(createWordList, linkOptions);
createWordList.LinkTo(filterWordList, linkOptions);
filterWordList.LinkTo(findReversedWords, linkOptions);
findReversedWords.LinkTo(printReversedWords, linkOptions);
// </snippet4>
// <snippet6>
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/cache/epub/16452/pg16452.txt");
// </snippet6>
// <snippet7>
// Mark the head of the pipeline as complete.
downloadString.Complete();
// </snippet7>
// <snippet8>
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
// </snippet8>
}
public ITargetBlock <Tuple <int> > CreateCacheUpdatePipeline(int padding)
{
SortedList <int, GopRecord> l_gopList = new SortedList <int, GopRecord>();
SortedList <int, FrameRecord> l_frameCache = new SortedList <int, FrameRecord>();
List <int> l_gopsLoaded = new List <int>();
int l_padding = padding;
int l_currentGOP = -1;
FrameRecord frame;
// initialize cache
bool success1 = true;
if (GetGopList(m_mp4Filename, l_gopList))
{
for (int i = 0; i < padding + 1; i++)
{
if (LoadGopIntoCache(i, l_gopList, l_frameCache))
{
l_gopsLoaded.Add(i);
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Error Intializing Cache", null));
success1 = false;
break;
}
}
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Error Intializing GOP List", null));
success1 = false;
}
// send first image in file
if (success1)
{
l_currentGOP = 0;
double timestamp;
int width, height, depth;
byte[] frameData;
if (l_frameCache.TryGetValue(0, out frame))
{
if (ReadFile(frame.filename, out timestamp, out width, out height, out depth, out frameData))
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.FRAME, "",
new FramePackage(0, timestamp, 0, 0.0, new DNNTools.ImagePackage(frameData, timestamp, width, height, depth))));
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Failed to read Image File from Cache: " +
frame.filename, null));
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
var RequestImage = new ActionBlock <Tuple <int> >(inputData =>
{
int frameIndex = inputData.Item1;
bool success;
double timestamp;
int width, height, depth;
byte[] frameData;
if (l_frameCache.TryGetValue(frameIndex, out frame))
{
success = ReadFile(frame.filename, out timestamp, out width, out height, out depth, out frameData);
if (success)
{
GopRecord gop = m_gopList[l_currentGOP];
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.FRAME, "",
new FramePackage(frameIndex, timestamp, l_currentGOP, gop.timestamp, new DNNTools.ImagePackage(frameData, timestamp, width, height, depth))));
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Failed to read Image File from Cache: " +
frame.filename, null));
}
}
else
{
// figure out which GOPs to load
int gopToLoad = GetGopIndexContainingFrameIndex(frameIndex, l_gopList);
if (gopToLoad != -1) // if gop found, -1 indicates that it wasn't found
{
if (LoadGopIntoCache(gopToLoad, l_gopList, l_frameCache))
{
l_gopsLoaded.Add(gopToLoad);
l_gopsLoaded.Sort();
if (l_frameCache.TryGetValue(frameIndex, out frame))
{
if (ReadFile(frame.filename, out timestamp, out width, out height, out depth, out frameData))
{
l_currentGOP = gopToLoad;
GopRecord gop = m_gopList[l_currentGOP];
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.FRAME, "",
new FramePackage(frameIndex, timestamp, l_currentGOP, gop.timestamp, new DNNTools.ImagePackage(frameData, timestamp, width, height, depth))));
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Failed to read Image File from Cache: " +
frame.filename + "\nafter trying to load GOP into Cache.", null));
}
}
// check to see if cache has grown to be too large. If so, remove a gop from cache
if (l_gopsLoaded.Count > (2 * l_padding + 1))
{
int lowestGop = l_gopsLoaded[0];
int highestGop = l_gopsLoaded[l_gopsLoaded.Count - 1];
int interval1 = Math.Abs(l_currentGOP - lowestGop);
int interval2 = Math.Abs(highestGop - l_currentGOP);
if (interval1 > interval2)
{
if (RemoveGopFromCache(lowestGop, l_gopList, l_frameCache))
{
l_gopsLoaded.RemoveAt(0);
}
}
else
{
if (RemoveGopFromCache(highestGop, l_gopList, l_frameCache))
{
l_gopsLoaded.RemoveAt(l_gopsLoaded.Count - 1);
}
}
}
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.ERROR, "Failed to Load GOP " + gopToLoad.ToString(), null));
}
}
else
{
OnVideoCacheEvent(new VideoCache_EventArgs(VideoCache_Status_Type.REACHED_END_OF_FILE, "Reached EOF", null));
}
}
});
return(RequestImage);
}
public SQLiteDatabase()
{
try
{
_maintenanceScheduler = new ActionBlock <bool>(async _ => await PerformDatabaseMaintenanceAsync(), new ExecutionDataflowBlockOptions {
BoundedCapacity = 2
});
_messageQueue = new AsynchronousMessageQueue(this, new[] { ContentDirectoryMessaging.CHANNEL });
_messageQueue.MessageReceived += OnMessageReceived;
_messageQueue.Start();
_settings = ServiceRegistration.Get <ISettingsManager>().Load <SQLiteSettings>();
_settings.LogSettings();
ServiceRegistration.Get <ISettingsManager>().Save(_settings);
LogVersionInformation();
if (_settings.EnableTraceLogging)
{
_sqliteDebugLogger = FileLogger.CreateFileLogger(ServiceRegistration.Get <IPathManager>().GetPath(@"<LOG>\SQLiteDebug.log"), LogLevel.Debug, false, true);
SQLiteLog.Initialize();
SQLiteLog.RemoveDefaultHandler();
SQLiteLog.Log += MPSQLiteLogEventHandler;
}
// We use our own collation sequence which is registered here to be able
// to sort items taking into account culture specifics
SQLiteFunction.RegisterFunction(typeof(SQLiteCultureSensitiveCollation));
var pathManager = ServiceRegistration.Get <IPathManager>();
string dataDirectory = pathManager.GetPath("<DATABASE>");
string databaseFile = Path.Combine(dataDirectory, _settings.DatabaseFileName);
// We use an URI instead of a simple database path and filename. The reason is that
// only this way we can switch on the shared cache mode of SQLite in System.Data.SQLite
// However, when using an URI, SQLite ignores the page size value specified in the connection string.
// Therefore we have to use a workaround below to create a database file with the specified page size.
string databaseFileForUri = databaseFile.Replace('\\', '/');
string databaseUri = System.Web.HttpUtility.UrlPathEncode("file:///" + databaseFileForUri + "?cache=shared");
_connectionPool = new ConnectionPool <SQLiteConnection>(CreateOpenAndInitializeConnection);
// We are using the ConnectionStringBuilder to generate the connection string
// This ensures code compatibility in case of changes to the SQLite connection string parameters
// More information on the parameters can be found here: http://www.sqlite.org/pragma.html
var connBuilder = new SQLiteConnectionStringBuilder
{
// Name of the database file including path as URI
FullUri = databaseUri,
// Use SQLite database version 3.x
Version = 3,
// Store GUIDs as binaries, not as string
// Saves some space in the database and is said to make search queries on GUIDs faster
BinaryGUID = true,
DefaultTimeout = _settings.LockTimeout,
CacheSize = _settings.CacheSizeInPages,
// Use the Write Ahead Log mode
// In this journal mode write locks do not block reads
// Needed to prevent sluggish behaviour of MP2 client when trying to read from the database (through MP2 server)
// while MP2 server writes to the database (such as when importing shares)
// More information can be found here: http://www.sqlite.org/wal.html
JournalMode = SQLiteJournalModeEnum.Wal,
// Do not use the inbuilt connection pooling of System.Data.SQLite
// We use our own connection pool which is faster.
Pooling = false,
// Sychronization Mode "Normal" enables parallel database access while at the same time preventing database
// corruption and is therefore a good compromise between "Off" (more performance) and "On"
// More information can be found here: http://www.sqlite.org/pragma.html#pragma_synchronous
SyncMode = SynchronizationModes.Normal,
// MP2's database backend uses foreign key constraints to ensure referential integrity.
// SQLite supports this, but it has to be enabled for each database connection by a PRAGMA command
// For details see http://www.sqlite.org/foreignkeys.html
ForeignKeys = true
};
if (_settings.EnableTraceLogging)
{
connBuilder.Flags = SQLiteConnectionFlags.LogAll;
}
_connectionString = connBuilder.ToString();
ServiceRegistration.Get <ILogger>().Info("SQLiteDatabase: Connection String used: '{0}'", _connectionString);
if (!File.Exists(databaseFile))
{
ServiceRegistration.Get <ILogger>().Info("SQLiteDatabase: Database file does not exists. Creating database file");
if (!Directory.Exists(dataDirectory))
{
Directory.CreateDirectory(dataDirectory);
}
// Since we use an URI in the standard connection string and system.data.sqlite therefore
// ignores the page size value in the connection string, this is a workaroung to make sure
// the page size value as specified in the settings is used. When the database file does
// not yet exists, we create a special connection string where we additionally set the
// datasource (which overrides the URI) and the page size. We then create a connection with
// that special connection string, open it and close it. That way the database file is created
// with the desired page size. The page size is permanently stored in the database file so that
// it is used when as of now we use connections with URI.
connBuilder.DataSource = databaseFile;
connBuilder.PageSize = _settings.PageSize;
using (var connection = new SQLiteConnection(connBuilder.ToString()))
{
connection.Open();
connection.Close();
}
}
// The following is necessary to avoid the creation of of a shared memory index file
// ("-shm"-file) when using exclusive locking mode. When WAL-mode is used, it is possible
// to switch between normal and exclusive locking mode at any time. However, the creation
// of a "-shm"-file can only be avoided, when exclusive locking mode is set BEFORE entering
// WAL-mode. If this is the case, it is not possible to leave exclusive locking mode
// without leaving WAL-mode before, because the "-shm"-file was not created.
// The regular connections in our connection pool use WAL-mode. Therefore we have
// to open one connection without WAL-Mode (here with JournalMode=OFF) and set locking_mode=
// EXCLUSIVE before we create the first regular connection that goes into the pool.
// To use exclusive locking mode, it is additionally necessary to set locking_mode=EXCLUSIVE
// for every connection in the pool via the InitializationCommand. If "PRAGMA locking_mode=
// EXCLUSIVE" is not in the InitializationCommand, normal locking mode is used
// although we issue "PRAGMA locking_mode=EXCLUSIVE" at this point.
// For details see here: http://sqlite.org/wal.html#noshm
// Avoiding the creation of an "-shm"-file materially improves the database performance.
if (_settings.UseExclusiveMode)
{
connBuilder.JournalMode = SQLiteJournalModeEnum.Off;
using (var connection = new SQLiteConnection(connBuilder.ToString()))
{
connection.Open();
using (var command = new SQLiteCommand(SQLiteSettings.EXCLUSIVE_MODE_COMMAND, connection))
command.ExecuteNonQuery();
connection.Close();
}
}
// Just test one "regular" connection, which is the first connection in the pool
using (var transaction = BeginTransaction())
transaction.Rollback();
}
catch (Exception e)
{
ServiceRegistration.Get <ILogger>().Critical("SQLiteDatabase: Error establishing database connection", e);
throw;
}
}
public void Reset()
{
_builder?.Complete();
_builder = null;
}
static async Task Main(string[] args)
{
Console.WriteLine("Start");
var option = new ExecutionDataflowBlockOptions {
BoundedCapacity = 2
};
var option1 = new ExecutionDataflowBlockOptions {
BoundedCapacity = 2, MaxDegreeOfParallelism = 3
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var downloader = new TransformBlock <string, byte[]>(async url =>
{
using (var http = new HttpClient())
{
var data = await http.GetByteArrayAsync(url).ConfigureAwait(false);
return(data);
}
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 10
});
var toImage = new TransformBlock <byte[], Image <Rgba32> >(buffer => Image.Load(buffer));
var grayscale = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
image.Mutate(x => x.Grayscale());
return(image);
});
var oilPaint = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+OP ");
image.Mutate(x => x.OilPaint());
Console.Write("-OP ");
return(image);
}, option1);
var pixelate = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+PX ");
image.Mutate(x => x.Pixelate());
Console.Write("-PX ");
return(image);
}, option);
var blur = new TransformBlock <Image <Rgba32>, Image <Rgba32> >(image =>
{
Console.Write("+BL ");
image.Mutate(x => x.GaussianBlur());
Console.Write("-BL ");
return(image);
}, option1);
var save = new ActionBlock <Image <Rgba32> >(async image =>
{
using (var f = new FileStream($"{Guid.NewGuid():N}.jpg",
FileMode.Create, FileAccess.Write, FileShare.None, 4096,
FileOptions.Asynchronous))
using (var mem = new MemoryStream())
{
image.SaveAsJpeg(mem);
mem.Position = 0;
await mem.CopyToAsync(f).ConfigureAwait(false);
}
});
toImage.LinkTo(grayscale, linkOptions);
grayscale.LinkTo(oilPaint, linkOptions);
grayscale.LinkTo(blur, linkOptions);
grayscale.LinkTo(pixelate, linkOptions);
pixelate.LinkTo(save);
oilPaint.LinkTo(save);
blur.LinkTo(save);
Task _ = Task.WhenAll(pixelate.Completion, oilPaint.Completion, blur.Completion)
.ContinueWith(c => save.Complete());
for (int i = 0; i < 30; i++)
{
downloader.Post(URL);
}
await Task.Delay(4000).ConfigureAwait(false);
downloader.LinkTo(toImage);
downloader.Complete();
await save.Completion.ConfigureAwait(false);
Console.WriteLine("Done");
Console.ReadKey();
}
public static void Start()
{
const int bc = 1;
// Download a book as a string
var downloadBook = new TransformBlock <string, string>(uri =>
{
Console.WriteLine("Downloading the book...");
return(new WebClient().DownloadString(uri));
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
// splits text into an array of strings.
var createWordList = new TransformBlock <string, string[]>(text =>
{
Console.WriteLine("Creating list of words...");
// Remove punctuation
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
{
tokens[i] = ' ';
}
}
text = new string(tokens);
return(text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries));
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
// Remove short words and return the count
var filterWordList = new TransformBlock <string[], int>(words =>
{
Console.WriteLine("Counting words...");
var wordList = words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
return(wordList.Count());
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = bc
});
var printWordCount = new ActionBlock <int>(wordcount =>
{
Console.WriteLine("Found {0} words",
wordcount);
});
downloadBook.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(printWordCount);
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadBook.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)createWordList).Fault(t.Exception);
}
else
{
createWordList.Complete();
}
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)filterWordList).Fault(t.Exception);
}
else
{
filterWordList.Complete();
}
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)printWordCount).Fault(t.Exception);
}
else
{
printWordCount.Complete();
}
});
// Download Origin of Species
downloadBook.Post("http://www.gutenberg.org/files/2009/2009.txt");
downloadBook.Post("http://www.gutenberg.org/files/2010/2010.txt");
downloadBook.Post("http://www.gutenberg.org/files/2011/2011.txt");
// Mark the head of the pipeline as complete.
downloadBook.Complete();
printWordCount.Completion.Wait();
Console.WriteLine("Finished. Press any key to exit.");
Console.ReadLine();
}
internal TrainEventProcessorFactory(OnlineTrainerSettingsInternal settings, Learner trainer, PerformanceCounters performanceCounters)
{
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (trainer == null)
{
throw new ArgumentNullException(nameof(trainer));
}
if (performanceCounters == null)
{
throw new ArgumentNullException(nameof(performanceCounters));
}
this.trainer = trainer;
this.performanceCounters = performanceCounters;
this.telemetry = new TelemetryClient();
this.telemetry.Context.Component.Version = GetType().Assembly.GetName().Version.ToString();
this.evalOperation = new EvalOperation(settings, performanceCounters);
this.latencyOperation = new LatencyOperation();
this.deserializeBlock = new TransformManyBlock <PipelineData, PipelineData>(
(Func <PipelineData, IEnumerable <PipelineData> >) this.Stage1_Deserialize,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4, // Math.Max(2, Environment.ProcessorCount - 1),
BoundedCapacity = 1024
});
this.deserializeBlock.Completion.Trace(this.telemetry, "Stage 1 - Deserialization");
this.learnBlock = new TransformManyBlock <object, object>(
(Func <object, IEnumerable <object> >) this.Stage2_ProcessEvent,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = 1024
});
this.learnBlock.Completion.Trace(this.telemetry, "Stage 2 - Learning");
// trigger checkpoint checking every second
this.checkpointTrigger = Observable.Interval(TimeSpan.FromSeconds(1))
.Select(_ => new CheckpointEvaluateTriggerEvent())
.Subscribe(this.learnBlock.AsObserver());
this.checkpointBlock = new ActionBlock <object>(
this.trainer.Checkpoint,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = 4
});
this.learnBlock.Completion.Trace(this.telemetry, "Stage 3 - CheckPointing");
// setup pipeline
this.deserializeBlock.LinkTo(
this.learnBlock,
new DataflowLinkOptions {
PropagateCompletion = true
});
this.learnBlock.LinkTo(
this.evalOperation.TargetBlock,
new DataflowLinkOptions {
PropagateCompletion = true
},
obj => obj is TrainerResult);
this.learnBlock.LinkTo(
this.checkpointBlock,
new DataflowLinkOptions {
PropagateCompletion = true
},
obj => obj is CheckpointData);
// consume all unmatched
this.learnBlock.LinkTo(DataflowBlock.NullTarget <object>());
}
public async Task RunPipelineAsync(CancellationToken cancellationToken)
{
if (parser.Document.SelectToken("$.imageRegistryCredentials") is JArray imageRegistryCredentials)
{
foreach (JObject imageRegistryCredential in imageRegistryCredentials)
{
HandleObjReplacement(imageRegistryCredential);
logger.LogInformation("using {@imageRegistryCredential}", imageRegistryCredential);
}
}
foreach (var part in parser.Document.SelectToken("$.pipe"))
{
var command = part.SelectToken("$.command");
var dependsOn = part.SelectToken("$.dependsOn")?.Select(c => c.ToString()).ToArray();
var parallelBlocker = int.MaxValue;
var parallelKey = part.SelectToken("$.name").ToString();
if (part.SelectToken("$.parallelContext") is JObject parallelContext)
{
HandleObjReplacement(parallelContext);
parallelBlocker = parallelContext.SelectToken("$.max")?.ToObject <int>() ?? parallelBlocker;
parallelKey = parallelContext.SelectToken("$.contextKey")?.ToString() ?? parallelKey;
logger.LogInformation("using {@parallelContext}: {parallelKey} {parallelBlocker}", parallelContext, parallelKey, parallelBlocker);
}
var semaphoreObject = options.GetSemaphore(parallelKey, parallelBlocker); //
parser.Functions["parallelContext"] = (document, arguments) => JToken.FromObject(new { counter = semaphoreObject.RunningCounter });
var volumnes = new Dictionary <string, JObject>();
var dependenciesResolved = true;
if (dependsOn != null)
{
foreach (var dependency in dependsOn)
{
if (dependency.StartsWith("/volumes/"))
{
var volumeName = dependency.Substring("/volumes/".Length);
var volume = parser.Document.SelectToken($"$.volumes.{volumeName}")?.DeepClone();
if (volume is JObject volumeObj)
{
HandleObjReplacement(volumeObj);
if (volume.SelectToken("$.azureFileShare") is JObject azureFileShare)
{
var storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(azureFileShare.SelectToken("$.storageAccountName").ToString(), azureFileShare.SelectToken("$.storageAccountKey").ToString()), true);
await storageAccount.CreateCloudFileClient().GetShareReference(azureFileShare.SelectToken("$.shareName").ToString()).CreateIfNotExistsAsync();
}
if (volume.SelectToken("$.local") is JObject local)
{
var uniqueId = local.SelectToken("$.uniqueId")?.ToString();
if (!string.IsNullOrEmpty(uniqueId))
{
local.Replace(CreateMD5(uniqueId));
}
}
//if (volume.SelectToken("$.mountedFrom").ToString().StartsWith("["))
//{
// volume.SelectToken("$.mountedFrom").Replace(parser.Evaluate(volume.SelectToken("$.mountedFrom").ToString()));
//}
// volumnes.Add(volume.SelectToken("$.mountedAt").ToString(),new EmptyStruct { });
volumnes.Add(volumeName, volumeObj);
}
else
{
dependenciesResolved = false;
break;
}
}
else
{
var completed = parser.Document.SelectToken($"$.pipe[?(@.name == '{dependency}')].outputs.completed")?.ToObject <bool>() ?? false;
if (!completed)
{
dependenciesResolved = false;
break;
}
}
}
if (!dependenciesResolved)
{
continue;
}
}
parser.Functions["mountedAt"] = (d, a) => volumnes[a.First().ToString()]["mountedAt"].ToString();
parser.Functions["mountedFrom"] = (d, a) => volumnes[a.First().ToString()]["mountedFrom"].ToString();
var commands = new List <string[]>()
{
};
var loop = part.SelectToken("loop");
if (loop != null)
{
if (loop.Type == JTokenType.String && loop.ToString().StartsWith("["))
{
loop = parser.Evaluate(loop.ToString());
}
logger.LogInformation("using {@loop}", loop);
foreach (var loopVar in loop)
{
parser.Functions["loop"] = (document, arguments) => arguments[0].ToString() == "var" ? loopVar: Path.GetFileName(loopVar.ToString());
var skip = part.SelectToken("$.skip");
if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>())
{
commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray());
}
}
}
else
{
var skip = part.SelectToken("$.skip");
if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>())
{
commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray());
}
}
if (!command.Any())
{
var outputs = part["outputs"] as JObject ?? new JObject();
outputs["completed"] = true;
part["outputs"] = outputs;
Console.WriteLine(part.ToString());
continue;
}
var image = part.SelectToken("$.image").ToString();
if (image.StartsWith("["))
{
part["image"] = parser.Evaluate(image).ToString();
}
var block = new ActionBlock <string[]>(async arguments =>
{
try
{
semaphoreObject.WaitOne();
var binds = volumnes.Select((v, i) => $"{GetMountedFrom(v)}:{v.Value.SelectToken("$.mountedAt").ToString()}").ToList();
var runCommand = $"docker run {string.Join(" ", binds.Select(b => $"-v {b}"))} {part.SelectToken("$.image").ToString()} {string.Join(" ", arguments)}";
logger.LogInformation(runCommand);
string stdOut = await executor.ExecuteStepAsync(parser, arguments, part, volumnes, cancellationToken);
parser.Functions["stdout"] = (d, o) => stdOut;
var outputs = part["outputs"] as JObject ?? new JObject();
foreach (var outputProperty in outputs.Properties())
{
if (outputProperty.Value.ToString().StartsWith("["))
{
outputProperty.Value.Replace(parser.Evaluate(outputProperty.Value.ToString()));
}
}
//Console.WriteLine("stdout:");
// Console.WriteLine(stdOut.ToString());
// await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters { RemoveVolumes = true, RemoveLinks = true, Force = true });
// Console.WriteLine(part.ToString());
}
finally
{
semaphoreObject.Release();
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = parser.Evaluate("[parameters('numberOfProcesses')]").ToObject <int>()
});
// var prop = command.Parent;
// Console.WriteLine(command.ToString());
foreach (var arguments in commands)
{
await block.SendAsync(arguments);
}
block.Complete();
await block.Completion;
{
var outputs = part["outputs"] as JObject ?? new JObject();
outputs["completed"] = true;
part["outputs"] = outputs;
logger.LogInformation("pipeline completed with {output}", outputs);
}
}
await executor.PipelineFinishedAsync(parser);
}
public void Init()
{
_massMail = new MassMail(Config.BlockSize, Config.UserAgent, Config.ConnectionString, Config.Mode);
_templateCache = new ConcurrentDictionary <long, Lazy <Template> >();
_attachmentCache = new ConcurrentDictionary <long, Lazy <Attach> >();
_dkimSignerCache = new ConcurrentDictionary <string, DkimSigner>();
_domailKeySignerCache = new ConcurrentDictionary <string, DomainKeySigner>();
//Get all private keys
GetDkimSigners();
//*** Create pipeline ***
//Create TransformBlock that gets table of client data and make a list of objects from them.
_parseXmlDataBlock = new TransformBlock <DataTable, List <Mail> >(sendData => ParseXmlData(sendData),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.ParseXmlMaxdop,
BoundedCapacity = Config.ParseXmlBufferSize
});
//Create TransformBlock that gets a list of client objects, send them email, and stores result in DataTable.
_sendEmailsBlock = new TransformBlock <List <Mail>, DataTable>(mails => SendEmails(_massMail, mails),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.SendEmailsMaxdop,
BoundedCapacity = Config.SendEmailsMaxdop
});
//Create BatchBlock that holds several DataTable and then propagates them out as an array.
_batchResultBlock = new BatchBlock <DataTable>(Config.BatchSize,
new GroupingDataflowBlockOptions
{
BoundedCapacity = Config.BatchSize
});
//Create ActionBlock that writes result into DB
_writeResultsBlock = new ActionBlock <DataTable[]>(results => WriteResults(_massMail, results),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
});
//*** Build pipeline ***
// POST --> _parseXmlDataBlock --> _sendEmailsBlock --> _batchResultBlock --> _writeResultsBlock
_parseXmlDataBlock.LinkTo(_sendEmailsBlock);
_sendEmailsBlock.LinkTo(_batchResultBlock);
_batchResultBlock.LinkTo(_writeResultsBlock);
_parseXmlDataBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_sendEmailsBlock).Fault(t.Exception);
}
else
{
_sendEmailsBlock.Complete();
}
});
_sendEmailsBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_batchResultBlock).Fault(t.Exception);
}
else
{
_batchResultBlock.Complete();
}
});
_batchResultBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)_writeResultsBlock).Fault(t.Exception);
}
else
{
_writeResultsBlock.Complete();
}
});
}
public void Start(ISenderCallback callback)
{
_callback = callback;
_sender = new ActionBlock <OutgoingMessageBatch>(SendBatch, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_sender.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_serializing = new ActionBlock <Envelope>(async e =>
{
try
{
await _batching.SendAsync(e);
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error while trying to serialize envelope {e}");
}
},
new ExecutionDataflowBlockOptions
{
CancellationToken = _cancellation,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_serializing.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_batchWriting = new TransformBlock <Envelope[], OutgoingMessageBatch>(
envelopes =>
{
var batch = new OutgoingMessageBatch(Destination, envelopes);
_queued += batch.Messages.Count;
return(batch);
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded, MaxDegreeOfParallelism = 10, CancellationToken = _cancellation
});
_batchWriting.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
_batchWriting.LinkTo(_sender);
_batching = new BatchingBlock <Envelope>(200, _batchWriting, _cancellation);
_batching.Completion.ContinueWith(x =>
{
if (x.IsFaulted)
{
_logger.LogException(x.Exception);
}
}, _cancellation);
}
public virtual void LinkTo(TClient client, Action <TCacheEventArgs> callback = null)
{
Throw.IfNull(client, nameof(client));
if (_isLinked)
{
if (client == Client)
{
return; // ignore.
}
throw new InvalidOperationException($"{GetType().Name} is linked to another {Client.GetType().Name}.");
}
_isLinked = true;
_callback = callback;
_bufferBlock = new BufferBlock <TEventArgs>(new DataflowBlockOptions
{
EnsureOrdered = true,
CancellationToken = Token,
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded
});
_actionBlock = new ActionBlock <TEventArgs>(async @event =>
{
TCacheEventArgs eventArgs = null;
try
{
eventArgs = await OnAction(@event)
.ConfigureAwait(false);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
Logger?.LogError(e, $"{GetType().Name}: Unhandled {nameof(OnAction)} exception.");
}
if (eventArgs != null)
{
try
{
_callback?.Invoke(eventArgs);
Update?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { }
catch (Exception e)
{
Logger?.LogError(e, $"{GetType().Name}: Unhandled update event handler exception.");
}
}
}, new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1,
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = Token,
SingleProducerConstrained = true
});
_bufferBlock.LinkTo(_actionBlock);
}
public Actor(Action <string, Exception> onException)
{
_onException = onException;
_block = new ActionBlock <Action>(x => Process(x));
}
/// <summary>
/// SimilarMediaHead(no)RTで始まるコマンドを処理してオブジェクトに詰め込む
/// ついでに類似画像も取得してオブジェクトに詰め込む
/// </summary>
/// <param name="cmd">SimilarMediaHead(no)RT で始まるSQL文を含むMySqlCommand</param>
/// <param name="login_user_id"></param>
/// <param name="SimilarLimit">取得する類似画像の上限</param>
/// <param name="GetIsolated">「ぼっち画像」を含める</param>
/// <param name="GetSimilars">与えられたツイに対して類似画像を取得するかどうか</param>
/// <returns></returns>
async Task <SimilarMediaTweet[]> TableToTweet(MySqlCommand cmd, long?login_user_id, int SimilarLimit, bool GetIsolated = false, bool GetSimilars = true)
{
ActionBlock <SimilarMediaTweet> GetSimilarsBlock = null;
if (GetSimilars)
{
GetSimilarsBlock = new ActionBlock <SimilarMediaTweet>(async(rettmp) =>
{
//この中でTableToTweet()が呼ばれる
rettmp.Similars = await SimilarMedia(rettmp.media.media_id, SimilarLimit, (rettmp.tweet.retweet ?? rettmp.tweet).tweet_id, login_user_id).ConfigureAwait(false);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
}
var TweetList = new List <SimilarMediaTweet>();
await ExecuteReader(cmd, (r) =>
{
SimilarMediaTweet rettmp = new SimilarMediaTweet();
rettmp.tweet.user.user_id = r.GetInt64(0);
rettmp.tweet.user.name = r.GetString(1);
rettmp.tweet.user.screen_name = r.GetString(2);
rettmp.tweet.user.local_profile_image_url = r.GetString(3);
rettmp.tweet.user.isprotected = r.GetBoolean(4);
rettmp.tweet.user.isprotected = r.GetBoolean(5);
rettmp.tweet.tweet_id = r.GetInt64(6);
rettmp.tweet.created_at = DateTimeOffset.FromUnixTimeSeconds(r.GetInt64(7));
rettmp.tweet.text = r.IsDBNull(8) ? null :r.GetString(8);
rettmp.tweet.favorite_count = r.GetInt32(9);
rettmp.tweet.retweet_count = r.GetInt32(10);
rettmp.tweet.text_html = LocalText.TextToLink(rettmp.tweet.text);
//アイコンが鯖内にあってもなくてもそれの絶対パスに置き換える
rettmp.tweet.user.local_profile_image_url = LocalText.ProfileImageUrl(rettmp.tweet.user, r.GetBoolean(4));
if (!r.IsDBNull(11)) //RTなら元ツイートが入っている
{
rettmp.tweet.retweet = new TweetData._tweet();
rettmp.tweet.retweet.tweet_id = r.GetInt64(11);
rettmp.tweet.retweet.user.user_id = r.GetInt64(12);
rettmp.tweet.retweet.user.name = r.GetString(13);
rettmp.tweet.retweet.user.screen_name = r.GetString(14);
rettmp.tweet.retweet.user.local_profile_image_url = r.GetString(15);
rettmp.tweet.retweet.user.is_default_profile_image = r.GetBoolean(16);
rettmp.tweet.retweet.user.isprotected = r.GetBoolean(17);
rettmp.tweet.retweet.created_at = DateTimeOffset.FromUnixTimeSeconds(r.GetInt64(18));
rettmp.tweet.retweet.text = r.GetString(19);
rettmp.tweet.retweet.favorite_count = r.GetInt32(20);
rettmp.tweet.retweet.retweet_count = r.GetInt32(21);
rettmp.tweet.retweet.text_html = LocalText.TextToLink(rettmp.tweet.retweet.text);
//アイコンが鯖内にあってもなくてもそれの絶対パスに置き換える
rettmp.tweet.retweet.user.local_profile_image_url = LocalText.ProfileImageUrl(rettmp.tweet.retweet.user, r.GetBoolean(16));
}
rettmp.media.media_id = r.GetInt64(22);
rettmp.media.orig_media_url = r.GetString(23);
rettmp.media.type = r.GetString(24);
rettmp.media.local_media_url = LocalText.MediaUrl(rettmp.media);
rettmp.SimilarMediaCount = r.IsDBNull(25) ? -1 : r.GetInt64(25); //COUNTはNOT NULLじゃない
rettmp.ExistsMoreMedia = SimilarLimit < rettmp.SimilarMediaCount;
TweetList.Add(rettmp);
if (GetSimilars)
{
GetSimilarsBlock.Post(rettmp);
}
}).ConfigureAwait(false);
if (GetSimilars)
{
GetSimilarsBlock.Complete();
await GetSimilarsBlock.Completion.ConfigureAwait(false);
if (GetIsolated)
{
return(TweetList.ToArray());
}
else
{
return(TweetList.Where(rettmp => rettmp.Similars.Length > 0).ToArray());
}
}
else
{
return(TweetList.ToArray());
}
}
public void Start()
{
queue = new ActionBlock <MessageProcessorHandle>(OnProcess);
}
static void Main(string[] args)
{
var queueName = "general_queue";
var exchangeName = "general_exchange";
var routingKey = "*";
var cancelSource = new CancellationTokenSource();
var poster1 = new RabbitPostBlock <RabbitMessage>(exchangeName, queueName, routingKey, cancelSource.Token);
var numMessages = 100000;
int consumeParallelism = 10;
var messages = new Dictionary <int, Tuple <RabbitMessage, Stopwatch> >();
for (int i = 0; i < numMessages; i++)
{
messages.Add(i, new Tuple <RabbitMessage, Stopwatch>(new RabbitMessage
{
BodyStr = $"Y'ello world {i}",
Properties = new Dictionary <string, object>()
{
{ "Index", i }
}
}, new Stopwatch()));
}
int receivedCount = 0;
var consumeBlock = new ActionBlock <RabbitMessage>(m =>
{
//if (m.Properties != null)
//{
// var i = (int)m.Properties["Index"];
// messages[i].Item2.Stop();
//}
//else Console.WriteLine(m.BodyStr);
receivedCount++;
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = consumeParallelism
});
var consumers = new List <RabbitRecieveBlock <RabbitMessage> >(consumeParallelism);
for (int i = 0; i < consumeParallelism; i++)
{
consumers.Add(new RabbitRecieveBlock <RabbitMessage>(exchangeName, queueName, routingKey, consumeBlock, true, cancelSource.Token));
}
foreach (var msg in messages.Values)
{
msg.Item2.Start();
poster1.PostMessageBlock.Post(msg.Item1);
}
while (receivedCount != numMessages)
{
Task.Delay(100, cancelSource.Token).Wait();
}
poster1.Dispose();
Task.WhenAll(consumers.Select(c => c.Stop())).Wait();
consumeBlock.Complete();
consumeBlock.Completion.Wait();
double totalTime = 0;
double minTime = double.MaxValue;
double maxTime = 0;
foreach (var msg in messages)
{
var elapsed = msg.Value.Item2.Elapsed.TotalMilliseconds;
totalTime += elapsed;
minTime = Math.Min(minTime, elapsed);
maxTime = Math.Max(maxTime, elapsed);
//Console.WriteLine($"{msg.Key} {msg.Value.Item2.Elapsed}");
}
Console.WriteLine($"Total: {totalTime}");
Console.WriteLine($"Average: {totalTime / numMessages}");
Console.WriteLine($"Min: {minTime}");
Console.WriteLine($"Max: {maxTime}");
}
public async Task TestCloning()
{
// Test cloning when a clone function is provided
{
int data = 42;
var bb = new BroadcastBlock <int>(x => - x);
Assert.True(bb.Post(data));
for (int i = 0; i < 3; i++)
{
Assert.Equal(expected: -data, actual: await bb.ReceiveAsync());
Assert.Equal(expected: -data, actual: bb.Receive());
Assert.Equal(expected: -data, actual: await bb.ReceiveAsync());
IList <int> items;
Assert.True(((IReceivableSourceBlock <int>)bb).TryReceiveAll(out items));
Assert.Equal(expected: items.Count, actual: 1);
Assert.Equal(expected: -data, actual: items[0]);
}
int result = 0;
var target = new ActionBlock <int>(i => {
Assert.Equal(expected: 0, actual: result);
result = i;
Assert.Equal(expected: -data, actual: i);
});
bb.LinkTo(target, new DataflowLinkOptions {
PropagateCompletion = true
});
bb.Complete();
await target.Completion;
}
// Test successful processing when no clone function exists
{
var data = new object();
var bb = new BroadcastBlock <object>(null);
Assert.True(bb.Post(data));
object result;
for (int i = 0; i < 3; i++)
{
Assert.Equal(expected: data, actual: await bb.ReceiveAsync());
Assert.Equal(expected: data, actual: bb.Receive());
Assert.Equal(expected: data, actual: await bb.ReceiveAsync());
IList <object> items;
Assert.True(((IReceivableSourceBlock <object>)bb).TryReceiveAll(out items));
Assert.Equal(expected: 1, actual: items.Count);
Assert.Equal(expected: data, actual: items[0]);
}
result = null;
var target = new ActionBlock <object>(o => {
Assert.Null(result);
result = o;
Assert.Equal(expected: data, actual: o);
});
bb.LinkTo(target, new DataflowLinkOptions {
PropagateCompletion = true
});
bb.Complete();
await target.Completion;
}
}
private void StartReceivingAccessReports()
{
var fifoName = ReportsFifoPath;
// opening FIFO for reading (blocks until there is at least one writer connected)
LogDebug($"Opening FIFO '{fifoName}' for reading");
using var readHandle = IO.Open(fifoName, IO.OpenFlags.O_RDONLY, 0);
if (readHandle.IsInvalid)
{
LogError($"Opening FIFO {fifoName} for reading failed");
return;
}
// make sure that m_lazyWriteHandle has been created
Analysis.IgnoreResult(m_lazyWriteHandle.Value);
// action block where parsing and processing of received bytes is offloaded (so that the read loop below is as tight as possible)
var actionBlock = new ActionBlock <byte[]>(ProcessBytes, new ExecutionDataflowBlockOptions
{
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxDegreeOfParallelism = 1,
EnsureOrdered = true
});
while (true)
{
// read length
byte[] messageLengthBytes = new byte[sizeof(int)];
var numRead = Read(readHandle, messageLengthBytes, 0, messageLengthBytes.Length);
if (numRead == 0) // EOF
{
break;
}
if (numRead < 0) // error
{
LogError($"Read from FIFO {ReportsFifoPath} failed with return value {numRead}");
break;
}
// decode length
int messageLength = BitConverter.ToInt32(messageLengthBytes, startIndex: 0);
// read a message of that length
byte[] messageBytes = new byte[messageLength];
numRead = Read(readHandle, messageBytes, 0, messageBytes.Length);
if (numRead < messageBytes.Length)
{
LogError($"Read from FIFO {ReportsFifoPath} failed with return value {numRead}");
break;
}
LogDebug($"Received a {numRead}-byte message");
// Add message to processing queue
actionBlock.Post(messageBytes);
}
// complete action block and wait for it to finish
actionBlock.Complete();
actionBlock.Completion.GetAwaiter().GetResult();
// report process tree completed
var report = new AccessReport
{
Operation = FileOperation.OpProcessTreeCompleted,
PathOrPipStats = AccessReport.EncodePath("")
};
Process.PostAccessReport(report);
}
/// <summary>
/// これを定期的に呼んで再接続やFriendの取得をやらせる
/// 1分経過しても終わらないときは打ち切る
/// 一度も取得していないアカウント→取得待ちが多い順 に処理する
/// </summary>
public async Task ConnectStreamers()
{
//アカウントが1個も割り当てられなくなってたら自殺する
if (!await db.ExistThisPid().ConfigureAwait(false))
{
Environment.Exit(1);
}
var LastCrawlTime = new Dictionary <long, long>();
while (ConnectBlockProceeeded.TryTake(out var proceeded))
{
//ツイートの取得時刻を保存する(たぶん)
//ここで(前回のConnectStreamers()で)REST取得したやつとUser streamに接続済みのやつだけ処理する(もうないけど)
if (proceeded.LastReceivedTweetId != 0 || proceeded.NeedConnect() == UserStreamer.NeedConnectResult.StreamConnected)
{
LastCrawlTime[proceeded.Token.UserId] = proceeded.LastMessageTime.ToUnixTimeSeconds();
}
}
var StoreLastCrawlTimeTask = db.StoreCrawlInfo_Timeline(LastCrawlTime);
Counter.PrintReset();
//TLがある程度進んでいるアカウントと一定時間取得してないアカウントだけ接続する
var now = DateTimeOffset.UtcNow;
var StreamersSelected = Streamers.Select(s => s.Value)
.Where(s => s.EstimatedTweetToReceive >= config.crawl.StreamSpeedTweets ||
(now - s.LastMessageTime).TotalSeconds > config.crawl.MaxRestInterval)
.OrderByDescending(s => s.EstimatedTweetToReceive)
.ThenBy(s => s.LastMessageTime)
.ToArray();
// UserStreamerの再接続やRESTによるツイート取得などを行うやつ
var ConnectBlock = new ActionBlock <UserStreamer>(async(Streamer) =>
{
try
{
var NeedConnect = Streamer.NeedConnect();
//VerifyCredentialsを実行して成功したらtrue
bool RevokeCheckSuccess = false;
//初回とRevoke疑いのときだけVerifyCredentials()する
//プロフィールを取得したい
if (NeedConnect == UserStreamer.NeedConnectResult.Postponed)
{
return;
}
else if (NeedConnect == UserStreamer.NeedConnectResult.First || RevokeRetryUserID.ContainsKey(Streamer.Token.UserId))
{
switch (await Streamer.VerifyCredentials().ConfigureAwait(false))
{
case UserStreamer.TokenStatus.Locked:
Streamer.PostponeConnect(); return;
case UserStreamer.TokenStatus.Revoked:
MarkRevoked(Streamer); return;
case UserStreamer.TokenStatus.Failure:
return;
case UserStreamer.TokenStatus.Success:
RevokeCheckSuccess = true;
break;
}
}
//Streamに接続したりRESTだけにしたり
if (NeedConnect == UserStreamer.NeedConnectResult.StreamConnected)
{
if (Streamer.NeedStreamSpeed() == UserStreamer.NeedStreamResult.RestOnly)
{
Streamer.DisconnectStream(); return;
}
else
{
Counter.ActiveStreamers.Increment();
}
}
else
{
//TLの速度を測定して必要ならStreamに接続
Counter.RestedStreamers.Increment();
switch (await Streamer.RestTimeline().ConfigureAwait(false))
{
case UserStreamer.TokenStatus.Revoked:
//長期間ログインされていないアカウントはVerifyCredentialsしか通らなくなるっぽい
//またVerifyCredentialsに失敗するまで放っとく
if (RevokeCheckSuccess)
{
Streamer.PostponeConnect();
}
//普通のRevoke疑い
else
{
MarkRevoked(Streamer);
}
break;
case UserStreamer.TokenStatus.Locked:
if (RevokeCheckSuccess)
{
UnmarkRevoked(Streamer);
}
Streamer.PostponeConnect();
break;
default:
if (RevokeCheckSuccess)
{
UnmarkRevoked(Streamer);
}
UserStreamer.NeedStreamResult NeedStream = Streamer.NeedStreamSpeed();
if (NeedStream == UserStreamer.NeedStreamResult.Stream)
{
Streamer.RecieveStream(); Counter.ActiveStreamers.Increment();
}
//DBが求めていればToken読み込み直後だけ自分のツイートも取得(初回サインイン狙い
if (Streamer.NeedRestMyTweet)
{
await Streamer.RestMyTweet().ConfigureAwait(false);
//User streamに繋がない場合はこっちでフォローを取得する必要がある
if (NeedStream != UserStreamer.NeedStreamResult.Stream)
{
await Streamer.RestFriend().ConfigureAwait(false);
}
await Streamer.RestBlock().ConfigureAwait(false);
Streamer.NeedRestMyTweet = false;
}
break;
}
}
ConnectBlockProceeeded.Add(Streamer);
}
catch (Exception e) { Console.WriteLine("ConnectBlock Faulted: {0}", e); }
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = ConnectBlockConcurrency,
BoundedCapacity = ConnectBlockConcurrency + 1, //これでもawaitする
SingleProducerConstrained = true,
});
var connectStopWatch = Stopwatch.StartNew();
bool connectTimedout = false;
using (var cancel = new CancellationTokenSource(60000))
{
try
{
foreach (var s in StreamersSelected)
{
if (!await ConnectBlock.SendAsync(s, cancel.Token).ConfigureAwait(false))
{
break;
} //そんなバナナ
//ツイートが詰まってたら休む
while (UserStreamerStatic.NeedConnectPostpone() && !cancel.Token.IsCancellationRequested)
{
await Task.Delay(1000, cancel.Token).ConfigureAwait(false);
}
}
ConnectBlock.Complete();
await ConnectBlock.Completion.ConfigureAwait(false);
}
catch (TaskCanceledException)
{
ConnectBlock.Complete();
connectTimedout = true;
}
}
//ConnectBlockの同時接続数を調整する
connectStopWatch.Stop();
int oldConnectBlockConcurrency = ConnectBlockConcurrency;
if (connectTimedout)
{
ConnectBlockConcurrency = config.crawl.MaxReconnectThreads;
}
else if (connectStopWatch.ElapsedMilliseconds < 40000)
{
ConnectBlockConcurrency = Math.Max(1, ConnectBlockConcurrency - 1);
}
else if (50000 < connectStopWatch.ElapsedMilliseconds)
{
ConnectBlockConcurrency = Math.Clamp((int)Math.Ceiling(ConnectBlockConcurrency * (connectStopWatch.ElapsedMilliseconds / 50000d)), 1, config.crawl.MaxReconnectThreads);
}
if (oldConnectBlockConcurrency != ConnectBlockConcurrency)
{
Console.WriteLine("ConnectBlockConcurrency: {0} -> {1}", oldConnectBlockConcurrency, ConnectBlockConcurrency);
}
await StoreLastCrawlTimeTask.ConfigureAwait(false);
//Revoke後再試行にも失敗したTokenはここで消す
await RemoveRevokedTokens().ConfigureAwait(false);
//接続が全然進まないときは殺してもらう
if (0 < Counter.ActiveStreamers.Get() + Counter.RestedStreamers.Get())
{
await WatchDogUdp.SendAsync(BitConverter.GetBytes(ThisPid), sizeof(int), WatchDogEndPoint).ConfigureAwait(false);
}
UserStreamerStatic.ShowCount();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
//_eventHandler.RegisterHandler("Monitor:ServersMonitorRecordAdded", (handler) => ServersMonitorRecorded += handler.Listener);
while (!stoppingToken.IsCancellationRequested)
{
var timerSeconds = _configuration.GetValue <int>("MonitorSettings:TrackerTimer", 15);
var nextTime = DateTime.Now;
var secondsLeft = timerSeconds - (nextTime.Second % timerSeconds);
nextTime = nextTime.AddTicks(-(nextTime.Ticks % TimeSpan.TicksPerSecond)); // We truncate it
nextTime = nextTime.AddSeconds(secondsLeft); // And we add the seconds left
TimeSpan difference = nextTime - DateTime.Now;
await Task.Delay(difference, stoppingToken);
_logger.LogDebug("Fetching all servers data to save it in the database");
using (var scope = _serviceProvider.CreateScope())
using (var db = scope.ServiceProvider.GetService <MonitorDBContext>())
{
var servers = db.Servers
.Include(s => s.Game)
.Include(s => s.Owner)
.Include(s => s.Group)
.Where(s => s.PID.HasValue);
//.ToListAsync();
List <ServerResourceMonitoringRegistry> monitoringEntries = new List <ServerResourceMonitoringRegistry>();
var cancellationToken = new CancellationTokenSource();
cancellationToken.CancelAfter(5000); // 5 seconds max
var writeCustomerBlock = new ActionBlock <Server>(async s =>
{
try
{
var usageTask = GetServerUsage(s);
var infoTask = _gameHandler.GetServerInfo(s, cancellationToken.Token);
await Task.WhenAll(usageTask, infoTask);
var usage = await usageTask;
var info = await infoTask;
var entry = new ServerResourceMonitoringRegistry()
{
Server = s,
TakenAt = DateTime.Now,
CPUUsage = usage.CPUUsage,
MemoryUsage = usage.MemoryUsage,
ActivePlayers = info.Players
};
await db.ServerResourceMonitoring.AddAsync(entry);
monitoringEntries.Add(entry);
}
catch (Exception err)
{
_logger.LogError(err, "There has been an error while fetching the data of server with id {0}", s.Id);
// Ignore, continue with the others
}
});
await servers.ForEachAsync(async s =>
{
await writeCustomerBlock.SendAsync(s);
});
writeCustomerBlock.Complete();
await writeCustomerBlock.Completion;
await db.SaveChangesAsync();
_logger.LogDebug("All servers information fetched and saved!");
cancellationToken.Dispose();
if (monitoringEntries.Count > 0)
{
ServersMonitorRecorded?.Invoke(this, new ServerRecordsEventArgs()
{
RowsInserted = monitoringEntries.Select(e => e.Id).ToArray()
});
}
}
}
}
public SafeActionBlock(ILogger logger, Func <TInput, Task> action, ExecutionDataflowBlockOptions dataFlowBlockOptions)
{
_logger = logger;
_action = action ?? throw new ArgumentNullException(nameof(action));
_actionBlock = new ActionBlock <TInput>(ActionAsync, dataFlowBlockOptions);
}
private bool CopyParallel(
CopyFileWithState copyFile,
int parallelism,
out List <ITaskItem> destinationFilesSuccessfullyCopied)
{
bool success = true;
// We must supply the same semantics as the single-threaded version above:
//
// - For copy operations in the list that have the same destination, we must
// provide for in-order copy attempts that allow re-copying different files
// and avoiding copies for later files that match SkipUnchangedFiles semantics.
// We must also add a destination file copy item for each attempt.
// - The order of entries in destinationFilesSuccessfullyCopied must match
// the order of entries passed in, along with copied metadata.
// - Metadata must not be copied to destination item if the copy operation failed.
//
// We split the work into different Tasks:
//
// - Entries with unique destination file paths each get their own parallel operation.
// - Each subset of copies into the same destination get their own Task to run
// the single-threaded logic in order.
//
// At the end we reassemble the result list in the same order as was passed in.
// Map: Destination path -> indexes in SourceFiles/DestinationItems array indices (ordered low->high).
var partitionsByDestination = new Dictionary <string, List <int> >(
DestinationFiles.Length, // Set length to common case of 1:1 source->dest.
StringComparer.OrdinalIgnoreCase);
for (int i = 0; i < SourceFiles.Length && !_cancellationTokenSource.IsCancellationRequested; ++i)
{
ITaskItem destItem = DestinationFiles[i];
string destPath = destItem.ItemSpec;
if (!partitionsByDestination.TryGetValue(destPath, out List <int> sourceIndices))
{
// Use 1 for list length - common case is for no destination overlap.
sourceIndices = new List <int>(1);
partitionsByDestination[destPath] = sourceIndices;
}
sourceIndices.Add(i);
}
// Lockless flags updated from each thread - each needs to be a processor word for atomicity.
var successFlags = new IntPtr[DestinationFiles.Length];
var actionBlockOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = parallelism,
CancellationToken = _cancellationTokenSource.Token
};
var partitionCopyActionBlock = new ActionBlock <List <int> >(
async(List <int> partition) =>
{
// Break from synchronous thread context of caller to get onto thread pool thread.
await System.Threading.Tasks.Task.Yield();
for (int partitionIndex = 0; partitionIndex < partition.Count && !_cancellationTokenSource.IsCancellationRequested; partitionIndex++)
{
int fileIndex = partition[partitionIndex];
ITaskItem sourceItem = SourceFiles[fileIndex];
ITaskItem destItem = DestinationFiles[fileIndex];
string sourcePath = sourceItem.ItemSpec;
// Check if we just copied from this location to the destination, don't copy again.
MSBuildEventSource.Log.CopyUpToDateStart(destItem.ItemSpec);
bool copyComplete = partitionIndex > 0 &&
String.Equals(
sourcePath,
SourceFiles[partition[partitionIndex - 1]].ItemSpec,
StringComparison.OrdinalIgnoreCase);
if (!copyComplete)
{
if (DoCopyIfNecessary(
new FileState(sourceItem.ItemSpec),
new FileState(destItem.ItemSpec),
copyFile))
{
copyComplete = true;
}
else
{
// Thread race to set outer variable but they race to set the same (false) value.
success = false;
}
}
else
{
MSBuildEventSource.Log.CopyUpToDateStop(destItem.ItemSpec, true);
}
if (copyComplete)
{
sourceItem.CopyMetadataTo(destItem);
successFlags[fileIndex] = (IntPtr)1;
}
}
},
actionBlockOptions);
foreach (List <int> partition in partitionsByDestination.Values)
{
bool partitionAccepted = partitionCopyActionBlock.Post(partition);
if (!partitionAccepted)
{
// Retail assert...
ErrorUtilities.ThrowInternalError("Failed posting a file copy to an ActionBlock. Should not happen with block at max int capacity.");
}
}
partitionCopyActionBlock.Complete();
partitionCopyActionBlock.Completion.GetAwaiter().GetResult();
// Assemble an in-order list of destination items that succeeded.
destinationFilesSuccessfullyCopied = new List <ITaskItem>(DestinationFiles.Length);
for (int i = 0; i < successFlags.Length; i++)
{
if (successFlags[i] != (IntPtr)0)
{
destinationFilesSuccessfullyCopied.Add(DestinationFiles[i]);
}
}
return(success);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary <string, ConcurrentDictionary <MetricInfo, int> >();
var windows = new ConcurrentDictionary <string, ConcurrentDictionary <MetricInfo, bool> >();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var timeWindow = new TimeWindow();
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var set = p as Set;
var metricName = set.Name + METRIC_IDENTIFIER_SEPARATOR + set.Value;
var metricInfo = new MetricInfo(metricName, set.Tags);
foreach (var period in timeWindow.AllPeriods)
{
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary <MetricInfo, bool>();
window.AddOrUpdate(metricInfo, (key2) => true, (key2, oldValue) => true);
return(window);
},
(key, window) =>
{
window.AddOrUpdate(metricInfo, (key2) => true, (key2, oldValue) => true);
return(window);
}
);
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var currentTimeWindow = new TimeWindow();
var periodsNotPresent = currentTimeWindow.GetDifferences(timeWindow);
// Make the current time window the one we use to manage the collections
timeWindow = currentTimeWindow;
CounterBucket bucket;
// (Parallel) For each period that was measured (Day, Week, Month etc)
// for every unique metric + value
// Count the number of entries that start with the metric name
// Add that metric to the list
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary <MetricInfo, bool> window;
// Take this window out of the dictionary
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary <MetricInfo, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Name.Split(METRIC_IDENTIFIER_SEPARATOR_SPLITTER, StringSplitOptions.RemoveEmptyEntries)[0] + qualifier;
var metricInfo = new MetricInfo(metricName, metricsAndValues[index].Key.Tags);
if (metrics.ContainsKey(metricInfo))
{
metrics[metricInfo] += 1;
}
else
{
metrics[metricInfo] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return(new KeyValuePair <MetricInfo, double>(
metric.Key,
metric.Value
));
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
break;
}
}
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public MessageObserver(Action <TMessage> action, string systemName)
{
_actionBlock = new ActionBlock <TMessage>(action);
_systemName = systemName;
Id = id++;
}
static void Main(string[] args)
{
// Test data
args = new string[] { "C:\\Development\\SonDar\\Paragon\\GuardAnalyzer", "Preview", "Default", "Example997*.cs" };
// arg0 : path to folder
string pathToStartFolder = args[0];
// arg1 : work mode
WorkMode mode;
if (!Enum.TryParse(args[1], out mode))
{
throw new Exception("Unknown Mode [args[1] = " + mode + "]");
}
// arg2 : changeModel file (Optional)
string path = Directory.GetCurrentDirectory() + "\\changes.txt";
if (args.Length > 2 && !args[2].Equals("Default"))
{
path = args[2];
}
// arg3 - wirdcard(Optional)
string wildcard = "*.cs";
if (args.Length > 3)
{
wildcard = args[3];
}
// Blocks list
TransformBlock <string, ArrayList> fileListBuilder = new TransformBlock <string, ArrayList>
(domainFolderName => (new FileListBuilder()).ParseDirectory(domainFolderName, wildcard));
TransformBlock <ArrayList, ChangeModel> guardAnalyzer = new TransformBlock <ArrayList, ChangeModel>
(fileList => (new GuardAnalyzer()).Prepare(fileList));
TransformBlock <string, ChangeModel> loader = new TransformBlock <string, ChangeModel>
(fileName => ChangeModel.Load(fileName));
TransformBlock <ChangeModel, ChangeModel> printer = new TransformBlock <ChangeModel, ChangeModel>
(model => model.Print());
ActionBlock <ChangeModel> writer = new ActionBlock <ChangeModel>(tempModel => tempModel.Save(path));
ActionBlock <ChangeModel> commiter = new ActionBlock <ChangeModel>(tempModel => (new GuardAnalyzer()).Commit(tempModel));
//Start
// Preview - fileListBuilder->guardAnalyzer->printer->writer
// Commit - loader->printer->commiter
// Forse - fileListBuilder->guardAnalyzer->printer->commiter
Logger.Log("Preview : ");
if (mode == WorkMode.Preview || mode == WorkMode.Force)
{
fileListBuilder.LinkTo(guardAnalyzer);
fileListBuilder.Completion.ContinueWith(task => guardAnalyzer.Complete());
guardAnalyzer.LinkTo(printer);
guardAnalyzer.Completion.ContinueWith(task => printer.Complete());
if (mode != WorkMode.Force)
{
printer.LinkTo(writer);
printer.Completion.ContinueWith(task => writer.Complete());
fileListBuilder.Post(pathToStartFolder);
fileListBuilder.Complete();
writer.Completion.Wait();
}
}
if (mode == WorkMode.Commit || mode == WorkMode.Force)
{
if (mode != WorkMode.Force)
{
loader.LinkTo(printer);
loader.Completion.ContinueWith(task => printer.Complete());
}
printer.LinkTo(commiter);
printer.Completion.ContinueWith(task => commiter.Complete());
if (mode != WorkMode.Force)
{
loader.Post(path);
loader.Complete();
}
else
{
fileListBuilder.Post(pathToStartFolder);
fileListBuilder.Complete();
}
commiter.Completion.Wait();
}
Console.ReadKey();
}
/// <inheritdoc/>
public async Task RunAsync(ProcessMode processMode, CancellationToken cancellationToken)
{
if (_messageEncoder == null)
{
throw new NotInitializedException();
}
try {
if (IsRunning)
{
return;
}
IsRunning = true;
_encodingBlock = new TransformManyBlock <DataSetMessageModel[], NetworkMessageModel>(
async input => {
try {
if (_dataSetMessageBufferSize == 1)
{
return(await _messageEncoder.EncodeAsync(input, _maxEncodedMessageSize).ConfigureAwait(false));
}
else
{
return(await _messageEncoder.EncodeBatchAsync(input, _maxEncodedMessageSize).ConfigureAwait(false));
}
}
catch (Exception e) {
_logger.Error(e, "Encoding failure");
return(Enumerable.Empty <NetworkMessageModel>());
}
},
new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken
});
_batchDataSetMessageBlock = new BatchBlock <DataSetMessageModel>(
_dataSetMessageBufferSize,
new GroupingDataflowBlockOptions {
CancellationToken = cancellationToken
});
_batchNetworkMessageBlock = new BatchBlock <NetworkMessageModel>(
_networkMessageBufferSize,
new GroupingDataflowBlockOptions {
CancellationToken = cancellationToken
});
_sinkBlock = new ActionBlock <NetworkMessageModel[]>(
async input => {
if (input != null && input.Any())
{
await _messageSink.SendAsync(input).ConfigureAwait(false);
}
else
{
_logger.Information("Sink block in engine {Name} triggered with empty input",
Name);
}
},
new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken
});
_batchDataSetMessageBlock.LinkTo(_encodingBlock);
_encodingBlock.LinkTo(_batchNetworkMessageBlock);
_batchNetworkMessageBlock.LinkTo(_sinkBlock);
_messageTrigger.OnMessage += MessageTriggerMessageReceived;
_messageTrigger.OnCounterReset += MessageTriggerCounterResetReceived;
if (_diagnosticInterval > TimeSpan.Zero)
{
_diagnosticsOutputTimer.Change(_diagnosticInterval, _diagnosticInterval);
}
await _messageTrigger.RunAsync(cancellationToken).ConfigureAwait(false);
}
finally {
IsRunning = false;
_messageTrigger.OnCounterReset -= MessageTriggerCounterResetReceived;
_messageTrigger.OnMessage -= MessageTriggerMessageReceived;
_diagnosticsOutputTimer.Change(Timeout.Infinite, Timeout.Infinite);
_batchTriggerIntervalTimer.Change(Timeout.Infinite, Timeout.Infinite);
await _sinkBlock.Completion;
}
}
public async Task ProcessFiles(string path, IProgress <ProgressReport> progress, RichTextBox txt_description)
{
int totalFilesFound = 0;
int totalFilesRead = 0;
int totalFilesHashed = 0;
int totalFilesUploaded = 0;
DateTime lastReported = DateTime.UtcNow;
void ReportProgress()
{
if (DateTime.UtcNow - lastReported < TimeSpan.FromSeconds(1)) //Try to fire only once a second, but this code is not perfect so you may get a few rapid fire.
{
return;
}
lastReported = DateTime.UtcNow;
var report = new ProgressReport(totalFilesFound, totalFilesRead, totalFilesHashed, totalFilesUploaded);
progress.Report(report);
}
var getFilesBlock = new TransformBlock <string, Dto>(filePath =>
{
var dto = new Dto(filePath, File.ReadAllBytes(filePath));
totalFilesRead++; //safe because single threaded.
return(dto);
});
var hashFilesBlock = new TransformBlock <Dto, Dto>(inDto =>
{
using (var md5 = System.Security.Cryptography.MD5.Create())
{
var hash = md5.ComputeHash(inDto.Data);
var outDto = new Dto(inDto.FilePath, hash, inDto.RelativePath, BitConverter.ToString(hash).Replace("-", ""));
Interlocked.Increment(ref totalFilesHashed); //Need the interlocked due to multithreaded.
ReportProgress();
return(outDto);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, BoundedCapacity = 50
});
var writeToDatabaseBlock = new ActionBlock <Dto>(arg =>
{
//Write to database here.
txt_description.AppendLine($"{arg.RelativePath} ({arg.MD5})");
// Main.AppendText(arg.FilePath);
totalFilesUploaded++;
ReportProgress();
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = 50
});
getFilesBlock.LinkTo(hashFilesBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
hashFilesBlock.LinkTo(writeToDatabaseBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var filePath in Directory.EnumerateFiles(path, "*.*", SearchOption.AllDirectories))
{
await getFilesBlock.SendAsync(filePath).ConfigureAwait(false);
totalFilesFound++;
ReportProgress();
}
getFilesBlock.Complete();
await writeToDatabaseBlock.Completion.ConfigureAwait(false);
ReportProgress();
}
public async Task DoNetwork()
{
OpenCL.IsEnabled = false;
var cacheFile = System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\stitch_cache.json";
State state = new State();
try
{
state = JsonConvert.DeserializeObject <State>(File.ReadAllText(cacheFile));
Console.WriteLine($"Using state from {cacheFile}");
} catch
{
Console.WriteLine("Couldn't load cache");
}
//state = new State();
AppState = state;
var workerPool = new LimitedConcurrencyLevelTaskScheduler(Math.Max(Environment.ProcessorCount - 2, 1)); // / 2, 1));
//var workerPool = new LimitedConcurrencyLevelTaskScheduler(1);
var snapshotState = new ActionBlock <State>((s) =>
{
var content = "";
s.Lock(lockedState => content = JsonConvert.SerializeObject(lockedState));
WriteAllTextWithBackup(cacheFile, content);
this.Dispatcher.Invoke(() => Joins.ItemsSource = AppState.Joins);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
state.ChangeListener = snapshotState;
var blockOptions = new ExecutionDataflowBlockOptions
{
TaskScheduler = workerPool,
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded // Handled by underlying scheduler
};
//var sourceDir = "C:/Users/Xavier/Source/ori-tracker/MapStitcher/Screenshots";
var sourceDir = "C:/Users/Xavier/Source/ori-tracker/MapStitcher/4KScreenshots";
/*
* IMagickImage image1 = new MagickImage(System.IO.Path.GetFullPath($"{sourceDir}/../Temp/forlorn-1.png"));
* image1 = image1.Clone();
* var settings = new MorphologySettings
* {
* Channels = Channels.Alpha,
* Method = MorphologyMethod.Distance,
* Kernel = Kernel.Euclidean,
* KernelArguments = "1,50!"
* };
*
* image1.Alpha(AlphaOption.Set);
* image1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
* image1.Morphology(settings);
* image1.Write(System.IO.Path.GetFullPath($"{sourceDir}/../Temp/forlorn-test.png"));
*/
/*
* MagickImage image1 = new MagickImage($"{sourceDir}/sorrow-1.png");
* MagickImage image2 = new MagickImage($"{sourceDir}/sorrow-2.png");
*/
/*
* var sourceFiles = new List<string>
* {
* $"{sourceDir}\\387290_20180314160604_1.png",
* };
*/
var sourceFiles = Directory.GetFiles(sourceDir, "*.png");
/*
* var sourceFiles = new List<string>
* {
* $"{sourceDir}\\forlorn-1.png",
* $"{sourceDir}\\forlorn-2.png",
* $"{sourceDir}\\forlorn-3.png",
* };
*/
//state.ClearJoins();
/*
* state.ClearNeedle(new NeedleKey { Key = $"{sourceDir}/forlorn-3.png", Gravity = Gravity.West });
* state.ClearJoin($"{sourceDir}/forlorn-2.png", $"{sourceDir}/forlorn-3.png");
* state.ClearJoin($"{sourceDir}/forlorn-1.png", $"{sourceDir}/forlorn-3.png");
* state.ClearJoin($"{sourceDir}/forlorn-2.png", $"{sourceDir}/forlorn-1.png");
*/
this.Dispatcher.Invoke(() => SourceImages.ItemsSource = SourceImages2.ItemsSource = sourceFiles);
this.Dispatcher.Invoke(() => Joins.ItemsSource = AppState.Joins);
UpdateUI();
var loadFromDiskBlock = new TransformBlock <string, string>(path =>
{
// TODO: Make this a load and crop task
var task = new StitchTask($"Load and crop {System.IO.Path.GetFileName(path)}");
this.Dispatcher.Invoke(() => Tasks.Add(task));
state.GetOrAddImage(path, () =>
{
var image = new MagickImage(path);
var originalSize = new Size(image.Width, image.Height);
int sideMargin = (int)(image.Width * 0.15); // The sides have a subtle animated mask over them. 280px wide on 1920px resolution. Crop them out.
int topMargin = (int)(image.Height * 0.17);
int bottomMargin = (int)(image.Height * 0.15);
var bounds = new MagickGeometry(sideMargin, topMargin, image.Width - sideMargin * 2, image.Height - bottomMargin - topMargin);
image.Crop(bounds);
image.RePage();
//image.Write("C:\\Users\\Xavier\\Source\\ori-tracker\\MapStitcher\\Temp\\" + System.IO.Path.GetFileName(path));
return(image);
});
task.Complete("Done", false);
return(path);
}, blockOptions);
var gravities = new TransformManyBlock <string, NeedleKey>(path =>
{
return(allGravities.Select(g => new NeedleKey()
{
Key = path, Gravity = g
}));
}, blockOptions);
var findNeedleBlock = new TransformBlock <NeedleKey, NeedleKey>(needle =>
{
var task = new FindNeedleTask(state, needle);
this.Dispatcher.Invoke(() => Tasks.Add(task));
task.Run();
return(needle);
}, blockOptions);
var findJoinBlock = new TransformBlock <SearchKey, string>(t =>
{
var haystack = t.Item1;
var needle = t.Item2;
var task = new SearchTask(state, haystack, needle);
this.Dispatcher.Invoke(() => Tasks.Add(task));
task.Run();
completedSearchTasks++;
return(haystack); // TODO: Figure out best thing to propagate. Maybe when match found?
}, blockOptions);
var broadcaster = new BroadcastBlock <string>(null);
var cartesian = new CartesianProductBlock <string, NeedleKey>();
var propagate = new DataflowLinkOptions {
PropagateCompletion = true
};
var headBlock = loadFromDiskBlock;
headBlock.LinkTo(broadcaster, propagate);
broadcaster.LinkTo(gravities, propagate);
gravities.LinkTo(findNeedleBlock, propagate);
// Don't propagate completion from left/right sources for cartesian join. It should
// complete when _both_ are done (which is it's default behaviour)
broadcaster.LinkTo(cartesian.Left, propagate);
findNeedleBlock.LinkTo(cartesian.Right, propagate);
var countTotals = new TransformManyBlock <Tuple <string, NeedleKey>, SearchKey>(t =>
{
var haystack = t.Item1;
var needle = t.Item2;
var none = Enumerable.Empty <SearchKey>();
if (haystack == needle.Key || !state.GetNeedle(needle).MeetsThreshold())
{
return(none);
}
var existingJoins = state.Joins;
var connectedJoins = new HashSet <HashSet <string> >();
foreach (var join in existingJoins)
{
var found = false;
foreach (var connectedSubset in connectedJoins)
{
if (connectedSubset.Contains(join.Image1) || connectedSubset.Contains(join.Image2))
{
connectedSubset.Add(join.Image1);
connectedSubset.Add(join.Image2);
found = true;
break;
}
}
if (!found)
{
var newSubset = new HashSet <string>();
newSubset.Add(join.Image1);
newSubset.Add(join.Image2);
connectedJoins.Add(newSubset);
}
}
connectedJoins.Aggregate(new HashSet <HashSet <string> >(), (acc, x) => {
var found = false;
foreach (var connectedSubset in acc)
{
if (connectedSubset.Overlaps(x))
{
connectedSubset.UnionWith(x);
found = true;
break;
}
}
if (!found)
{
acc.Add(x);
}
return(acc);
});
if (connectedJoins.Any(x => x.Contains(haystack) && x.Contains(needle.Key)))
{
Console.WriteLine("Two images already connected via transitive joins, skipping");
return(none);
}
totalSearchTasks++;
return(Enumerable.Repeat(SearchKey.Create(t.Item1, t.Item2), 1));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
cartesian.LinkTo(countTotals, propagate);
countTotals.LinkTo(findJoinBlock, propagate);
var sink = new ActionBlock <string>(s => { });
findJoinBlock.LinkTo(sink, propagate);
foreach (var file in sourceFiles)
{
headBlock.Post(file);
}
headBlock.Complete();
await sink.Completion.ContinueWith(_ =>
{
Console.WriteLine("Pipeline Finished");
/*
* this.Dispatcher.Invoke(() => TaskGrid.ItemsSource = Tasks.Where(x =>
* {
* if (x is SearchTask)
* {
* var task = (SearchTask)x;
* return task.Name.Contains("Found");
* return task.searchResult.MeetsThreshold();
* return task.searchResult.Distance < 2500;
* }
* return false;
* }));
*/
Dictionary <string, Point> completedJoins = new Dictionary <string, Point>();
var remainingJoins = new List <State.Join>(state.Joins);
var rejects = new List <State.Join>();
var images = new MagickImageCollection();
var lastCycleCount = 0;
var morphologySettings = new MorphologySettings
{
Channels = Channels.Alpha,
Method = MorphologyMethod.Distance,
Kernel = Kernel.Euclidean,
KernelArguments = "1,50!"
};
while (remainingJoins.Count > 0 && remainingJoins.Count != lastCycleCount)
{
lastCycleCount = remainingJoins.Count;
foreach (var join in remainingJoins)
{
if (completedJoins.Count == 0)
{
var tempPath = System.IO.Path.GetTempFileName();
var tempPath2 = System.IO.Path.GetTempFileName();
// Initial seed
var i1 = state.Image(join.Image1).Clone();
var i2 = state.Image(join.Image2).Clone();
i1.Alpha(AlphaOption.Set);
i1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i1.Morphology(morphologySettings);
i1.Write(tempPath);
i1.Dispose();
i1 = new MagickImage(tempPath);
i1.BackgroundColor = new MagickColor(18, 18, 18);
i2.Alpha(AlphaOption.Set);
i2.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i2.Morphology(morphologySettings);
i2.Write(tempPath2);
i2.Dispose();
i2 = new MagickImage(tempPath2);
i2.Page = new MagickGeometry($"{ToOffset(join.JoinPoint.X)}{ToOffset(join.JoinPoint.Y)}");
images.Add(i1);
images.Add(i2);
completedJoins.Add(join.Image1, new Point(0, 0));
completedJoins.Add(join.Image2, join.JoinPoint);
File.Delete(tempPath);
File.Delete(tempPath2);
}
else
{
Point offset = join.JoinPoint;
if (completedJoins.ContainsKey(join.Image1) && completedJoins.ContainsKey(join.Image2))
{
// NOOP
}
else if (completedJoins.ContainsKey(join.Image1))
{
completedJoins.TryGetValue(join.Image1, out offset);
var tempPath = System.IO.Path.GetTempFileName();
var i2 = state.Image(join.Image2).Clone();
var joinPoint = new Point(join.JoinPoint.X + offset.X, join.JoinPoint.Y + offset.Y);
i2.Alpha(AlphaOption.Set);
i2.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i2.Morphology(morphologySettings);
i2.Write(tempPath);
//i2.Dispose();
//i2 = new MagickImage(tempPath);
i2.Page = new MagickGeometry($"{ToOffset(joinPoint.X)}{ToOffset(joinPoint.Y)}");
images.Add(i2);
File.Delete(tempPath);
completedJoins.Add(join.Image2, joinPoint);
}
else if (completedJoins.ContainsKey(join.Image2))
{
completedJoins.TryGetValue(join.Image2, out offset);
var tempPath = System.IO.Path.GetTempFileName();
var i1 = state.Image(join.Image1).Clone();
var joinPoint = new Point(offset.X - join.JoinPoint.X, offset.Y - join.JoinPoint.Y);
i1.Alpha(AlphaOption.Set);
i1.VirtualPixelMethod = VirtualPixelMethod.Transparent;
i1.Morphology(morphologySettings);
i1.Write(tempPath);
//i1.Dispose();
//i1 = new MagickImage(tempPath);
i1.Page = new MagickGeometry($"{ToOffset(joinPoint.X)}{ToOffset(joinPoint.Y)}");
images.Add(i1);
File.Delete(tempPath);
completedJoins.Add(join.Image1, joinPoint);
}
else
{
rejects.Add(join);
}
}
}
remainingJoins = rejects.ToList();
rejects.Clear();
}
if (images.Any())
{
var merged = images.Merge();
//merged.BackgroundColor = new MagickColor(0, 0, 0);
//merged.Alpha(AlphaOption.Remove);
//merged.Alpha(AlphaOption.Off);
merged.Write("C:\\Users\\Xavier\\Source\\ori-tracker\\MapStitcher\\Temp\\map.png");
DisplayImage(Viewer, merged);
Console.WriteLine("Done Compositing");
}
});
}
public SubscribeManager(XProvider provider)
{
_provider = provider;
_action = new ActionBlock <Event>(SubscribeAction);
}
