public static void Generate(string root)
{
Directory.CreateDirectory("docs");
var _executingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
File.Copy(Path.Combine(_executingDirectory, "Resources", "Nocco.css"), Path.Combine("docs", "nocco.css"), true);
File.Copy(Path.Combine(_executingDirectory, "Resources", "prettify.js"), Path.Combine("docs", "prettify.js"), true);
var getFiles = DirectoryTraveler.Create();
var readFiles = FileReader.Create();
var redFileBuffer = new BufferBlock<FileContents>();
var parseFiles = FileParser.Create();
var renderCode = CodeRenderer.Create();
var renderDocs = DocRenderer.Create();
var generateHtml = HtmlGenerator.Create();
var persistanceBuffer = new BufferBlock<RenderedFile>();
var persistFile = FilePersister.Create();
var propCompl = new DataflowLinkOptions { PropagateCompletion = true };
getFiles.LinkTo(readFiles, propCompl);
readFiles.LinkTo(redFileBuffer, propCompl);
redFileBuffer.LinkTo(parseFiles, propCompl);
parseFiles.LinkTo(renderCode, propCompl);
renderCode.LinkTo(renderDocs, propCompl);
renderDocs.LinkTo(generateHtml, propCompl);
generateHtml.LinkTo(persistanceBuffer, propCompl);
persistanceBuffer.LinkTo(persistFile, propCompl);
getFiles.Post(root);
getFiles.Complete();
persistanceBuffer.Completion.Wait();
}
public async Task Process(GamePointer gp, List<PlayerState> players, BufferBlock<UpdatePacket> sendChannel)
{
players.Add(_player);
var squares = gp.Game.GetSquares().Select(x => new SquareDTO(x)).ToArray();
//FIXME: right now the order is important here, lest we lack player color information when rendering flags
var json2 = JSON.Serialize<PlayerState[]>(players.ToArray());
var packet2 = new UpdatePacket()
{
Type = "player",
Data = json2
};
var j2 = JSON.Serialize<UpdatePacket>(packet2);
await _uc.SendAsync(j2);
var json = JSON.Serialize<SquareDTO[]>(squares);
var packet = new UpdatePacket()
{
Type = "square",
Data = json
};
var j = JSON.Serialize<UpdatePacket>(packet);
await _uc.SendAsync(j);
}
public async Task Process(GamePointer gamePointer, List<PlayerState> players, BufferBlock<UpdatePacket> sendChannel)
{
if (_player.Dead)
{
return;
}
List<Square> changedSquares = gamePointer.Game.Update(_command, _player);
if (changedSquares.Count > 0)
{
var json = JSON.Serialize<Square[]>(changedSquares.ToArray());
var packet = new UpdatePacket()
{
Type = "square",
Data = json
};
await sendChannel.SendAsync(packet);
}
var player = JSON.Serialize<PlayerState>(_player);
var pp = new UpdatePacket()
{
Type = "player",
Data = player
};
await sendChannel.SendAsync(pp);
}
public SQLMessageQueue(IDbConnectionProvider connectionProvider, ISQLDialect dialect, QueueName queueName,
IQueueListener listener, QueueOptions options = default(QueueOptions))
{
if (connectionProvider == null) throw new ArgumentNullException("connectionProvider");
if (dialect == null) throw new ArgumentNullException("dialect");
if (queueName == null) throw new ArgumentNullException("queueName");
if (listener == null) throw new ArgumentNullException("listener");
_connectionProvider = connectionProvider;
_dialect = dialect;
_queueName = queueName;
_listener = listener;
_autoAcknowledge = options.AutoAcknowledge;
_maxAttempts = options.MaxAttempts <= 0 ? 10 : options.MaxAttempts;
_retryDelay = options.RetryDelay < TimeSpan.Zero ? TimeSpan.Zero : options.RetryDelay;
var concurrencyLimit = options.ConcurrencyLimit <= 0
? QueueOptions.DefaultConcurrencyLimit
: options.ConcurrencyLimit;
_concurrentMessageProcessingSlot = new SemaphoreSlim(concurrencyLimit);
_cancellationTokenSource = new CancellationTokenSource();
_queuedMessages = new BufferBlock<SQLQueuedMessage>(new DataflowBlockOptions
{
CancellationToken = _cancellationTokenSource.Token
});
}
public FilesystemMessageQueue(DirectoryInfo directory, IQueueListener listener,
QueueOptions options = default(QueueOptions))
{
if (directory == null) throw new ArgumentNullException("directory");
if (listener == null) throw new ArgumentNullException("listener");
_directory = directory;
_deadLetterDirectory = new DirectoryInfo(Path.Combine(directory.FullName, "dead"));
_listener = listener;
_autoAcknowledge = options.AutoAcknowledge;
_maxAttempts = options.MaxAttempts <= 0 ? 10 : options.MaxAttempts;
_retryDelay = options.RetryDelay < TimeSpan.Zero ? TimeSpan.Zero : options.RetryDelay;
var concurrencyLimit = options.ConcurrencyLimit <= 0
? QueueOptions.DefaultConcurrencyLimit
: options.ConcurrencyLimit;
_concurrentMessageProcessingSlot = new SemaphoreSlim(concurrencyLimit);
_cancellationTokenSource = new CancellationTokenSource();
_queuedMessages = new BufferBlock<MessageFile>(new DataflowBlockOptions
{
CancellationToken = _cancellationTokenSource.Token
});
}
public void ProduceLogs(int count, int buffSize)
{
var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize };
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.buff.log", false);
BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6);
writer.SendAsync(line).Wait();
}
buffer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
_waitHandle.Signal();
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock);
transformBlock.LinkTo(actionBlock);
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
_waitHandle.Wait();
}
public void Run()
{
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>();
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Signaling completion to the source block.");
bufferBlock.Complete();
Console.WriteLine("Done.");
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(200);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(500);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
Console.WriteLine("Waiting for the completion to be propagated through the network...");
actionBlock.Completion.ContinueWith(t =>
{
Console.WriteLine("Finished processing.");
Console.WriteLine(string.Format("Completion status: {0}.", t.Status));
}).Wait();
}
public async Task TestDataBroadcaster2()
{
var random = new Random();
var buffer = new BufferBlock<int>();
int sum1 = 0;
int sum2 = 0;
int sum3 = 0;
var action1 = new ActionBlock<int>(i => sum1 = sum1 + i);
var action2 = new ActionBlock<int>(i => sum2 = sum2 + i);
var action3 = new ActionBlock<int>(i => sum3 = sum3 + i);
buffer.ToDataflow().LinkToMultiple(new []{action1, action2, action3}.Select(a => a.ToDataflow()).ToArray());
for (int j = 0; j < 1000; j++)
{
buffer.Post((int)(random.NextDouble() * 10000));
}
buffer.Complete();
await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion, action3.Completion);
Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2);
Assert.AreEqual(sum1, sum2);
Assert.AreEqual(sum1, sum3);
}
public HttpNegotiationQueue(WebSocketFactoryCollection standards, WebSocketConnectionExtensionCollection extensions, WebSocketListenerOptions options)
{
Guard.ParameterCannotBeNull(standards, "standards");
Guard.ParameterCannotBeNull(extensions, "extensions");
Guard.ParameterCannotBeNull(options, "options");
_options = options;
_extensions = extensions;
_cancel = new CancellationTokenSource();
_semaphore = new SemaphoreSlim(options.ParallelNegotiations);
_sockets = new BufferBlock<Socket>(new DataflowBlockOptions()
{
BoundedCapacity = options.NegotiationQueueCapacity,
CancellationToken = _cancel.Token
});
_negotiations = new BufferBlock<WebSocketNegotiationResult>(new DataflowBlockOptions()
{
BoundedCapacity = options.NegotiationQueueCapacity,
CancellationToken = _cancel.Token,
});
_cancel.Token.Register(_sockets.Complete);
_cancel.Token.Register(_negotiations.Complete);
_handShaker = new WebSocketHandshaker(standards, _options);
Task.Run((Func<Task>)WorkAsync);
}
public void TestExceptionInLoadTxInput()
{
var expectedException = new Exception();
var coreStorage = new Mock<ICoreStorage>();
var chainedHeader = RandomData.RandomChainedHeader();
var tx = RandomData.RandomTransaction(new RandomDataOptions { TxInputCount = 1 });
var txLookupKey = new TxLookupKey(UInt256.Zero, 0);
var loadingTx = new LoadingTx(1, tx, chainedHeader, ImmutableArray.Create(txLookupKey));
var loadingTxes = new BufferBlock<LoadingTx>();
loadingTxes.Post(loadingTx);
loadingTxes.Complete();
// throw expected exception when the input transaction is looked up
Transaction outputTx = null;
coreStorage.Setup(x => x.TryGetTransaction(txLookupKey.BlockHash, txLookupKey.TxIndex, out outputTx)).Throws(expectedException);
var loadedTxes = TxLoader.LoadTxes(coreStorage.Object, loadingTxes);
Exception actualEx;
AssertMethods.AssertAggregateThrows<Exception>(() =>
loadedTxes.ToEnumerable().ToList(), out actualEx);
Assert.AreSame(expectedException, actualEx);
}
public void ReceiveCompletedTest ()
{
var block = new BufferBlock<int> ();
block.Complete ();
AssertEx.Throws<InvalidOperationException> (
() => block.Receive (TimeSpan.FromMilliseconds (1000)));
}
public void TestUtxoLookAheadWithTransactions()
{
var deferredCursor = new Mock<IDeferredChainStateCursor>();
deferredCursor.Setup(x => x.CursorCount).Returns(4);
var blockTxes = new BufferBlock<DecodedBlockTx>();
var lookAhead = UtxoLookAhead.LookAhead(blockTxes, deferredCursor.Object);
var blockTx0 = BlockTx.Create(0, RandomData.RandomTransaction(new RandomDataOptions { TxInputCount = 2 }));
var blockTx1 = BlockTx.Create(1, RandomData.RandomTransaction(new RandomDataOptions { TxInputCount = 2 }));
var blockTx2 = BlockTx.Create(2, RandomData.RandomTransaction(new RandomDataOptions { TxInputCount = 2 }));
blockTxes.Post(blockTx0);
blockTxes.Post(blockTx1);
blockTxes.Post(blockTx2);
blockTxes.Complete();
// verify each transaction was forwarded
var expectedBlockTxes = new[] { blockTx0, blockTx1, blockTx2 };
var warmedTxes = lookAhead.ReceiveAllAsync().Result;
Assert.AreEqual(3, warmedTxes.Count);
CollectionAssert.AreEqual(expectedBlockTxes.Select(x => x.Hash).ToList(), warmedTxes.Select(x => x.Hash).ToList());
// verify each non-coinbase input transaction was warmed up
var expectedLookups = expectedBlockTxes.Skip(1).SelectMany(x => x.Transaction.Inputs.Select(input => input.PrevTxOutputKey.TxHash));
foreach (var txHash in expectedLookups)
deferredCursor.Verify(x => x.WarmUnspentTx(txHash));
Assert.IsTrue(lookAhead.Completion.Wait(2000));
}
public void Run()
{
var cts = new CancellationTokenSource();
Console.WriteLine("Generating first {0} powers of 2.", MaxItems);
var bufferBlock = new BufferBlock<int>(new DataflowBlockOptions { CancellationToken = cts.Token });
Enumerable.Range(1, MaxItems)
.ToList()
.ForEach(i => bufferBlock.Post(i));
Console.WriteLine("Scheduling cancellation after 5 seconds.");
cts.CancelAfter(TimeSpan.FromSeconds(5));
Console.WriteLine("Creating and linking the remaing blocks to the network.");
var transformBlock = new TransformBlock<int, double>(i =>
{
Thread.Sleep(500);
return Math.Pow(2, i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 1, CancellationToken = cts.Token });
var actionBlock = new ActionBlock<double>(async i =>
{
await Task.Delay(1000);
Console.WriteLine(i);
}, new ExecutionDataflowBlockOptions { BoundedCapacity = 10, CancellationToken = cts.Token });
bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
var t1 = bufferBlock.Completion.ContinueWith(t => Console.WriteLine("Buffer block status: {0}", t.Status));
var t2 = actionBlock.Completion.ContinueWith(t => Console.WriteLine("Action block status: {0}", t.Status));
Console.WriteLine("Waiting for the network to finish.");
Task.WaitAll(t1, t2);
}
protected WebSocketClient()
{
_sendBuffer = new BufferBlock<string>(new DataflowBlockOptions
{
BoundedCapacity = 8
});
}
public void TestCaptureOrder()
{
// post source ints
var intSource = new BufferBlock<int>();
intSource.Post(1);
intSource.Post(2);
intSource.Post(99);
intSource.Post(98);
intSource.Complete();
// capture source order
var orderedInts = OrderingBlock.CaptureOrder<int, long, long>(
intSource, intValue => (long)intValue);
// post longs to combine, in reverse of original order
var longSource = new BufferBlock<long>();
longSource.Post(99);
longSource.Post(98);
longSource.Post(2);
longSource.Post(1);
longSource.Complete();
// apply source order
var orderedLongs = orderedInts.ApplyOrder(longSource, longValue => longValue);
// verify the original order was preserved
CollectionAssert.AreEqual(new long[] { 1, 2, 99, 98 }, orderedLongs.ToEnumerable().ToList());
}
public static IPropagatorBlock<File, FileLine> GetFileLinesEnumeratorBlock()
{
var resultsBlock = new BufferBlock<FileLine>();
var actionBlock = new ActionBlock<File>(
async file =>
{
using (var reader = new System.IO.StreamReader(new System.IO.FileStream(
file.FullPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.Read,
bufferSize: 4096,
useAsync: true)))
{
string line;
var row = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
resultsBlock.Post(new FileLine(file, row, line));
}
row++;
}
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public void MultipleBindingTest ()
{
BufferBlock<int> buffer = new BufferBlock<int> ();
var evt = new CountdownEvent (10);
int count = 0;
ActionBlock<int> block = new ActionBlock<int> ((i) => { Interlocked.Decrement (ref count); evt.Signal (); });
IDisposable bridge = buffer.LinkTo (block);
for (int i = 0; i < 10; i++)
Assert.IsTrue (buffer.Post (i));
evt.Wait ();
Assert.AreEqual (-10, count);
count = 0;
evt.Reset ();
bridge.Dispose ();
ActionBlock<int> block2 = new ActionBlock<int> ((i) => { Interlocked.Increment (ref count); evt.Signal (); });
buffer.LinkTo (block2);
for (int i = 0; i < 10; i++)
Assert.IsTrue (buffer.Post (i));
evt.Wait ();
Assert.AreEqual (10, count);
}
public ExecutionPipeline(Kernel kernel)
{
_kernel = kernel;
_commandQueue = new BufferBlock<CommandRequest[]>();
_queryQueue = new BatchBlock<QueryRequest>(MaxConcurrentQueries);
var transactionHandler = new ActionBlock<object>(t =>
{
if (t is QueryRequest[])
{
var queries = t as QueryRequest[];
Task[] tasks = queries.Select(q => Task.Factory.StartNew(_ => ExecuteQuery(q), null)).ToArray();
Task.WaitAll(tasks);
}
else if (t is CommandRequest[])
{
var commands = t as CommandRequest[];
foreach (var commandContext in commands)
{
var result = _kernel.Execute(commandContext.Command);
commandContext.Response.Post(result);
}
}
});
_commandQueue.LinkTo(transactionHandler);
_queryQueue.LinkTo(transactionHandler);
_timer = new Timer(_ => _queryQueue.TriggerBatch());
_timer.Change(Interval, Interval);
}
public void RaceTest ()
{
var source1 = new BufferBlock<int> ();
var source2 = new BufferBlock<int> ();
int action1 = 0;
int action2 = 0;
var completion = DataflowBlock.Choose (
source1, _ => action1++,
source2, _ => action2++);
var barrier = new Barrier (2);
var t1 = Task.Run (() =>
{
barrier.SignalAndWait ();
source1.Post (10);
});
var t2 = Task.Run (() =>
{
barrier.SignalAndWait ();
source2.Post (20);
});
Task.WaitAll (t1, t2);
Assert.IsTrue (completion.Wait (1000));
Assert.AreEqual (1, action1 + action2);
int item;
Assert.IsTrue (source1.TryReceive (out item) || source2.TryReceive (out item));
}
public void GreedyJoin3Test ()
{
var scheduler = new TestScheduler ();
var block =
new JoinBlock<int, int, int> (new GroupingDataflowBlockOptions
{ TaskScheduler = scheduler });
var source1 =
new BufferBlock<int> (new DataflowBlockOptions { TaskScheduler = scheduler });
var source2 =
new BufferBlock<int> (new DataflowBlockOptions { TaskScheduler = scheduler });
var source3 =
new BufferBlock<int> (new DataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsNotNull (source1.LinkTo (block.Target1));
Assert.IsNotNull (source2.LinkTo (block.Target2));
Assert.IsNotNull (source3.LinkTo (block.Target3));
Assert.IsTrue (source1.Post (1));
scheduler.ExecuteAll ();
int i;
Assert.IsFalse (source1.TryReceive (out i));
Assert.IsTrue (source2.Post (11));
Assert.IsTrue (source3.Post (21));
scheduler.ExecuteAll ();
Assert.IsFalse (source2.TryReceive (out i));
Assert.IsFalse (source3.TryReceive (out i));
Tuple<int, int, int> tuple;
Assert.IsTrue (block.TryReceive (out tuple));
Assert.AreEqual (Tuple.Create (1, 11, 21), tuple);
}
public void Start()
{
var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink);
var source = new BufferBlock<GetPageMessage>();
var linkOptions = new DataflowLinkOptions {PropagateCompletion = false};
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var worker = new TransformBlock<GetPageMessage, PageResultMessage>(
(Func<GetPageMessage, PageResultMessage>)Worker, options);
source.LinkTo(worker, linkOptions);
worker.LinkTo(sink, linkOptions);
}
foreach (var url in UrlList.Urls)
{
source.Post(new GetPageMessage{ Url = url });
}
source.Complete();
sink.Completion.Wait();
}
public static IPropagatorBlock<FileLinesEnumerator.FileLine, FileLineWord> GetLineSplitterBlock()
{
var resultsBlock = new BufferBlock<FileLineWord>();
var actionBlock = new ActionBlock<FileLinesEnumerator.FileLine>(
l =>
{
int? wordStart = null;
var endOfProcesssing = false;
for (var col = 1; !endOfProcesssing; ++col)
{
endOfProcesssing = col > l.Line.Length;
var ch = endOfProcesssing ? ' ' : l.Line[col - 1];
if (char.IsLetter(ch))
{
if (!wordStart.HasValue)
{
wordStart = col;
}
}
else if (wordStart.HasValue)
{
resultsBlock.Post(new FileLineWord(
l.File,
l.Line.Substring(wordStart.Value - 1, col - wordStart.Value).ToUpperInvariant(),
l.Row,
wordStart.Value));
wordStart = null;
}
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public ParallelPersistor(ISession session, int asyncWorkersCount)
{
_session = session;
_workerTasks = new Task[asyncWorkersCount];
_maximumQueueSize = asyncWorkersCount * 100;
_queriesWaitingInLineSemaphore = new SemaphoreSlim(_maximumQueueSize);
_insertionQueue = new BufferBlock<PendingInsert>();
}
public void WithElementsStillLingeringButFaulted ()
{
var block = new BufferBlock<int> ();
block.Post (42);
((IDataflowBlock)block).Fault (new Exception ());
Assert.IsTrue (block.Completion.IsCompleted);
Assert.AreEqual (TaskStatus.Faulted, block.Completion.Status);
}
public void ReceiveCancelledTest ()
{
var block = new BufferBlock<int> ();
var tokenSource = new CancellationTokenSource (200);
AssertEx.Throws<OperationCanceledException> (
() => block.Receive (tokenSource.Token));
}
public void Join(Player p)
{
lock (this)
{
var po = new BufferBlock<string>();
pmscp.Add(p, new MultiplayerScriptContextPlayer(p, po, OnJoin(p)));
}
}
static async Task PushUpdates(BufferBlock<UpdatePacket> sendQueue, ConnectionHub connectionHub)
{
while (true)
{
var next = await sendQueue.ReceiveAsync();
var json = JSON.Serialize<UpdatePacket>(next);
await connectionHub.Broadcast(json);
}
}
public static ISourceBlock<ValidatableTx> ReplayRollbackUtxo(ICoreStorage coreStorage, IChainState chainState, ChainedHeader replayBlock, CancellationToken cancelToken = default(CancellationToken))
{
// replaying rollback of an on-chain block, use the chainstate tx index for replay, same as replaying forward
if (chainState.Chain.BlocksByHash.ContainsKey(replayBlock.Hash))
{
return ReplayFromTxIndex(coreStorage, chainState, replayBlock, replayForward: false, cancelToken: cancelToken);
}
// replaying rollback of an off-chain (re-org) block, use the unminted information for replay
else
{
IImmutableList<UnmintedTx> unmintedTxesList;
if (!chainState.TryGetBlockUnmintedTxes(replayBlock.Hash, out unmintedTxesList))
{
//TODO if a wallet/monitor were to see a chainstate block that wasn't flushed to disk yet,
//TODO and if bitsharp crashed, and if the block was orphaned: then the orphaned block would
//TODO not be present in the chainstate, and it would not get rolled back to generate unminted information.
//TODO DeferredChainStateCursor should be used in order to re-org the chainstate in memory and calculate the unminted information
throw new MissingDataException(replayBlock.Hash);
}
var unmintedTxes = ImmutableDictionary.CreateRange(
unmintedTxesList.Select(x => new KeyValuePair<UInt256, UnmintedTx>(x.TxHash, x)));
var lookupLoadingTx = new TransformBlock<DecodedBlockTx, ValidatableTx>(
blockTx =>
{
var tx = blockTx.Transaction;
var txIndex = blockTx.Index;
var prevTxOutputs = ImmutableArray.CreateBuilder<PrevTxOutput>(!blockTx.IsCoinbase ? tx.Inputs.Length : 0);
if (!blockTx.IsCoinbase)
{
UnmintedTx unmintedTx;
if (!unmintedTxes.TryGetValue(tx.Hash, out unmintedTx))
throw new MissingDataException(replayBlock.Hash);
prevTxOutputs.AddRange(unmintedTx.PrevTxOutputs);
}
return new ValidatableTx(blockTx, replayBlock, prevTxOutputs.MoveToImmutable());
});
IEnumerator<BlockTx> blockTxes;
if (!coreStorage.TryReadBlockTransactions(replayBlock.Hash, out blockTxes))
{
throw new MissingDataException(replayBlock.Hash);
}
var blockTxesBuffer = new BufferBlock<DecodedBlockTx>();
blockTxesBuffer.LinkTo(lookupLoadingTx, new DataflowLinkOptions { PropagateCompletion = true });
blockTxesBuffer.SendAndCompleteAsync(blockTxes.UsingAsEnumerable().Select(x => x.Decode()).Reverse(), cancelToken).Forget();
return lookupLoadingTx;
}
}
public void CompletedImmediateTest()
{
var source = new BufferBlock<int>();
source.Complete();
var task = source.OutputAvailableAsync();
Assert.IsTrue(task.Wait(1000));
Assert.IsFalse(task.Result);
}
public FileBufferBlock(BufferBlock <WordStream> buffer, string fileName)
{
this._fileName = fileName;
this._buffer = buffer;
}
public CSVSource(string fileName) : this()
{
FileName = fileName;
Buffer = new BufferBlock <string[]>();
}
public void TestEtw()
{
RemoteExecutor.Invoke(() =>
{
using (var listener = new TestEventListener(new Guid("16F53577-E41D-43D4-B47E-C17025BF4025"), EventLevel.Verbose))
{
ActionBlock <int> ab = null;
BufferBlock <int> bb = null;
int remaining = 0;
CountdownEvent ce = new CountdownEvent(0);
// Check that block creation events fire
const int DataflowBlockCreatedId = 1;
remaining = 2;
listener.RunWithCallback(ev => {
Assert.Equal(expected: DataflowBlockCreatedId, actual: ev.EventId);
remaining--;
},
() => {
ab = new ActionBlock <int>(i => { });
bb = new BufferBlock <int>(); // trigger block creation event
Assert.Equal(expected: 0, actual: remaining);
});
// Check that linking events fire
const int BlockLinkedId = 4;
remaining = 1;
IDisposable link = null;
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockLinkedId, actual: ev.EventId);
remaining--;
},
() => {
link = bb.LinkTo(ab);
Assert.Equal(expected: 0, actual: remaining);
});
// Check that unlinking events fire
const int BlockUnlinkedId = 5;
remaining = 1;
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockUnlinkedId, actual: ev.EventId);
remaining--;
},
() => {
link.Dispose();
Assert.Equal(expected: 0, actual: remaining);
});
// Check that task launched events fire
const int TaskLaunchedId = 2;
ce.Reset(1);
listener.RunWithCallback(ev => {
Assert.Equal(expected: TaskLaunchedId, actual: ev.EventId);
ce.Signal();
},
() => {
ab.Post(42);
ce.Wait();
Assert.Equal(expected: 0, actual: ce.CurrentCount);
});
// Check that completion events fire
const int BlockCompletedId = 3;
ce.Reset(2);
listener.RunWithCallback(ev => {
Assert.Equal(expected: BlockCompletedId, actual: ev.EventId);
ce.Signal();
},
() => {
ab.Complete();
bb.Complete();
ce.Wait();
Assert.Equal(expected: 0, actual: ce.CurrentCount);
});
}
}).Dispose();
}
private static async Task ReadData <T>(ICollection <string> pathList, Func <string, string, T> factory,
ObjectPool <EventHubClient> pool, int randomSeed, AsyncConsole console,
CancellationToken cancellationToken, int waittime, DataFormat dataFormat)
where T : TaxiData
{
if (pathList == null)
{
throw new ArgumentNullException(nameof(pathList));
}
if (factory == null)
{
throw new ArgumentNullException(nameof(factory));
}
if (pool == null)
{
throw new ArgumentNullException(nameof(pool));
}
if (console == null)
{
throw new ArgumentNullException(nameof(console));
}
if (waittime > 0)
{
TimeSpan span = TimeSpan.FromMilliseconds(waittime);
await Task.Delay(span);
}
string typeName = typeof(T).Name;
Random random = new Random(randomSeed);
// buffer block that holds the messages . consumer will fetch records from this block asynchronously.
BufferBlock <T> buffer = new BufferBlock <T>(new DataflowBlockOptions()
{
BoundedCapacity = 100000
});
// consumer that sends the data to event hub asynchronoulsy.
var consumer = new ActionBlock <T>(
(t) =>
{
using (var client = pool.GetObject())
{
return(client.Value.SendAsync(new EventData(Encoding.UTF8.GetBytes(
t.GetData(dataFormat))), t.PartitionKey));
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 100000,
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = 100,
}
);
// link the buffer to consumer .
buffer.LinkTo(consumer, new DataflowLinkOptions()
{
PropagateCompletion = true
});
long messages = 0;
// iterate through the path list and act on each file from here on
foreach (var path in pathList)
{
ZipArchive archive = new ZipArchive(
File.OpenRead(path),
ZipArchiveMode.Read);
foreach (var entry in archive.Entries)
{
using (var reader = new StreamReader(entry.Open()))
{
var header = reader.ReadLines()
.First();
// Start consumer
var lines = reader.ReadLines()
.Skip(1);
// for each line , send to event hub
foreach (var line in lines)
{
await buffer.SendAsync(factory(line, header)).ConfigureAwait(false);
if (++messages % 10000 == 0)
{
// random delay every 10000 messages are buffered ??
await Task.Delay(random.Next(100, 1000))
.ConfigureAwait(false);
await console.WriteLine($"Created {messages} records for {typeName}").ConfigureAwait(false);
}
}
}
}
}
buffer.Complete();
await Task.WhenAll(buffer.Completion, consumer.Completion);
await console.WriteLine($"Created total {messages} records for {typeName}").ConfigureAwait(false);
}
/// <summary>
/// Will spawn a parallel job each group of files. A "group" is considered by
/// the name of the file excluding the extension. For example one set of
/// text files will be EPASomeName.001 -> up to EPASomeName.999 this
/// would be one group and the a second one would be EPABigInflate.001 -> EPABigInflate.999
/// </summary>
/// <param name="rootPath"></param>
/// <param name="destPath"></param>
/// <param name="fileFileter"></param>
public static void ProcessFileTreeToPDF(string rootPath, string destPath, string fileFileter = "*")
{
IProvideDataContent content;;
IEnumerable <IGrouping <string, string> > fg;;
if (!Directory.Exists(rootPath))
{
// must be a file
if (String.Compare(Path.GetExtension(rootPath), ".zip", true) == 0)
{
using (var zipFile = ZipFile.OpenRead(rootPath))
{
fg = zipFile.Entries.Select(e => e.FullName).GroupBy(e => Path.GetFileNameWithoutExtension(e));
}
content = new ZipDataContent(rootPath);
}
else
{
return;
}
}
else
{
var files = Directory.GetFiles(rootPath, searchPattern: fileFileter);
// Create a group of filenames by basename
fg = files.GroupBy(f => Path.GetFileNameWithoutExtension(f));
content = new FileDataContent(rootPath);
}
var buffer = new BufferBlock <InputDataHolder>();
//Parallel.ForEach(fg, g=> CreateCSVFiles(rootPath, destPath, g));
Task <int> consumeTask = null;
//should use delegates or interface here, write to a zip if passed a zip extension for the output
if (Path.GetExtension(destPath).ToLower() == ".zip")
{
consumeTask = ConsumeAsyncZip(buffer, destPath);
}
else if (Directory.Exists(destPath))
{
consumeTask = ConsumeAsyncPdf(buffer, destPath);
}
else
{
throw new Exception("Cannot decide to zip or csv to file");
}
// Start feeding the beast
var loopResult = Parallel.ForEach(fg, new ParallelOptions()
{
//MaxDegreeOfParallelism = Environment.ProcessorCount // put a limit because in some cases we have seen over parallization
}, g => {
var r = content.Process(g.Key, g.ToImmutableList()); //ProcessConversion(rootPath, g);
// inefficiently keep trying to post to the buffer, in case we get an error (shouldn't but...)
// also should handle exceptions better.
while (!buffer.Post(r.Result))
{
Console.Error.WriteLine("Could not post to buffer");
Task.Delay(500).Wait(); //wait for a half second
}
});
buffer.Complete(); // let the buffer know, there should be no more additions
var result = consumeTask.Result; // should be able to finish and drain the buffer
Console.WriteLine($"added {result} entries to {destPath}");
}
public EmailSenderHostedService(IConfiguration configuration, ILogger <EmailSenderHostedService> logger)
{
this.logger = logger;
this.mailMessages = new BufferBlock <MailMessage>();
this.smtpClient = CreateSmtpClient(configuration);
}
public BlockTransformation()
{
NLogger = NLog.LogManager.GetLogger("ETL");
InputData = new List <TInput>();
OutputBuffer = new BufferBlock <TInput>();
}
protected async Task ReceiveLoop(CancellationToken cancelToken, WebSocket socket, BufferBlock <object> sendQueue, string token)
{
using var memStream = new MemoryStream();
while (!cancelToken.IsCancellationRequested)
{
//NOTE: the ReceiveObjectAsync throws an exception on close
var receiveItem = await socket.ReceiveObjectAsync <WebSocketRequest>(memStream, cancelToken);
var userId = ValidateToken(token); // Validate every time
services.logger.LogDebug($"WS request '{receiveItem.id}'({receiveItem.type}) from {userId}");
var response = new WebSocketResponse()
{
id = receiveItem.id,
type = receiveItem.type,
requestUserId = userId
};
if (receiveItem.type == "ping")
{
response.data = new {
serverTime = DateTime.UtcNow
};
}
else if (receiveItem.type == "selfbroadcast")
{
response.data = receiveItem.data;
await BroadcastToSelfClients(userId, response);
//This skips the sending of the response, because in a broadcast, you'll receive it anyway
continue;
}
else if (receiveItem.type == "userlist")
{
response.data = await GetUserStatusesAsync(userId);
}
else if (receiveItem.type == "setuserstatus")
{
try
{
if (receiveItem.data == null)
{
throw new RequestException("Must set data to a dictionary of contentId:status");
}
var statuses = (((JObject)receiveItem.data).ToObject <Dictionary <string, string> >() ?? throw new RequestException("Couldn't parse sent userlist!"))
.ToDictionary(x => long.Parse(x.Key), y => y.Value);//(Dictionary<long, string>)receiveItem.data;
//TODO: this will need to do some magic to send the userlist to everyone. I suppose if I
//had a list of all waiters and their send queues.... hmmmm that would actually just work.
foreach (var status in statuses)
{
await AddUserStatusAsync(userId, status.Key, status.Value);
}
}
catch (Exception ex)
{
if (!(ex is RequestException))
{
services.logger.LogWarning($"Error when user {userId} set statuses to {receiveItem.data}: {ex}");
}
response.error = $"Error while setting statuses: {ex.Message}";
}
}
else if (receiveItem.type == "request")
{
try
{
if (receiveItem.data == null)
{
throw new RequestException("Must provide search criteria for request!");
}
var searchRequest = ((JObject)receiveItem.data).ToObject <SearchRequests>() ??
throw new RequestException("Couldn't parse search criteria!");
using (var search = services.dbFactory.CreateSearch())
{
var searchResult = await search.Search(searchRequest, userId);
response.data = searchResult;
}
}
catch (Exception ex)
{
response.error = $"Error during search: {ex.Message}";
}
}
else if (receiveItem.type == "write")
{
try
{
if (receiveItem.data == null)
{
throw new RequestException("Must provide write item for request!");
}
var writeData = ((JObject)receiveItem.data).ToObject <WebsocketWriteData>() ??
throw new RequestException("Couldn't parse write data! Must provide type, etc");
var writeObject = writeData.@object ?? throw new RequestException("Couldn't parse 'object' before type is known!"); //((JObject)writeData.@object);
var type = writeData.type;
//This sucks. Wonder if I can make it better
if (type == nameof(RequestType.message))
{
response.data = await WriteAsync(writeObject.ToObject <MessageView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.content))
{
response.data = await WriteAsync(writeObject.ToObject <ContentView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.user))
{
response.data = await WriteAsync(writeObject.ToObject <UserView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.uservariable))
{
response.data = await WriteAsync(writeObject.ToObject <UserVariableView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.watch))
{
response.data = await WriteAsync(writeObject.ToObject <WatchView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.vote))
{
response.data = await WriteAsync(writeObject.ToObject <VoteView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else if (type == nameof(RequestType.ban))
{
response.data = await WriteAsync(writeObject.ToObject <BanView>() ?? throw new RequestException($"Couldn't parse {type}!"), userId, writeData.activityMessage);
}
else
{
throw new RequestException($"Unknown write type {type}");
}
}
catch (Exception ex)
{
response.error = $"{ex.GetType().Name}: {ex.Message}";
}
}
else
{
response.error = $"Unknown request type {receiveItem.type}";
}
sendQueue.Post(response);
}
}
public async Task <ActionResult <string> > WebSocketListenAsync([FromQuery] string token, [FromQuery] int?lastId = null)
{
try
{
services.logger.LogDebug($"ws METHOD: {HttpContext.Request.Method}, HEADERS: " +
JsonConvert.SerializeObject(HttpContext.Request.Headers,
Formatting.None, new JsonSerializerSettings()
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
}));
if (!HttpContext.WebSockets.IsWebSocketRequest)
{
return(BadRequest("You must send a websocket request to this endpoint!"));
}
//None of these will throw exceptions that we can match anyway, so they'll bubble up accordingly...
//well that might not be entirely true if we're returning 500 errors specifically but... change it if you need.
using var cancelSource = new CancellationTokenSource();
using var dualCancel = CancellationTokenSource.CreateLinkedTokenSource(cancelSource.Token, appLifetime.ApplicationStopping, appLifetime.ApplicationStopped);
var sendQueue = new BufferBlock <object>();
List <Task> runningTasks = new List <Task>();
long userId = 0;
int realLastId = lastId == null?eventQueue.GetCurrentLastId() : lastId.Value;
using var socket = await HttpContext.WebSockets.AcceptWebSocketAsync(acceptContextGenerator ());
try
{
//ALWAYS add the sendloop first so we can process outgoing messages
runningTasks.Add(SendLoop(dualCancel.Token, socket, sendQueue));
//You want to keep this validation token thing inside the main exception handler, as ANY of the
//below tasks could throw the token validation exception!
userId = ValidateToken(token);
services.logger.LogInformation($"Websocket started for user {userId}");
//ALWAYS send the lastId message, it's basically our "this is the websocket and you're connected"
var response = new WebSocketResponse()
{
type = "lastId",
data = realLastId,
requestUserId = userId
};
sendQueue.Post(response);
if (!currentListeners.TryAdd(trackerId, new WebsocketListenerData()
{
userId = userId, sendQueue = sendQueue
}))
{
throw new InvalidOperationException("INTERNAL ERROR: couldn't add you to the listener array!");
}
//Can send and receive at the same time, but CAN'T send/receive multiple at the same time.
runningTasks.Add(ReceiveLoop(dualCancel.Token, socket, sendQueue, token));
runningTasks.Add(ListenLoop(dualCancel.Token, realLastId, sendQueue, token));
var completedTask = await Task.WhenAny(runningTasks);
runningTasks.Remove(completedTask);
await completedTask; //To throw the exception, if there is one
}
catch (OperationCanceledException ex)
{
services.logger.LogDebug($"Websocket was cancelled by system, we're probably shutting down: {ex.Message}");
// ALL should output an operation cancel exception but
// I'm ASSUMING that this will specifically not exit until they're ALL done...
try { await Task.WhenAll(runningTasks); }
catch (OperationCanceledException) { } //Fine
catch (Exception exi) { services.logger.LogError($"CRITICAL: EXCEPTION THROWN FROM CANCELED WEBSOCKET WAS UNEXPECTED TYPE: {exi}"); }
finally { runningTasks.Clear(); }
}
catch (TokenException ex)
{
//Note: it is OK to use sendQueue even if the sender loop isn't started, because we dump the
//remaining queue anyway in the finalizer
services.logger.LogError($"Token exception in websocket: {ex}");
sendQueue.Post(new WebSocketResponse()
{
type = "badtoken", error = ex.Message
});
}
catch (data.ClosedException ex)
{
services.logger.LogDebug($"User {userId} closed websocket on their end, this is normal: {ex.Message}");
}
//ALl other unhandled exceptions
catch (Exception ex)
{
services.logger.LogError("Unhandled exception in websocket: " + ex.ToString());
sendQueue.Post(new WebSocketResponse()
{
type = "unexpected", error = $"Unhandled exception: {ex}"
});
}
finally
{
if (runningTasks.Count > 0)
{
//Cause the cancel source to close naturally after 2 seconds, giving us enough time to send
//out remaining messages, but also allowing us to close immediately if everything was already completed
//(because we wait on the tasks themselves, which could complete earlier than the cancel)
cancelSource.CancelAfter(2000);
try { await Task.WhenAll(runningTasks); }
catch (ClosedException ex) { services.logger.LogDebug($"Client closed connection manually, this is normal!: {ex.Message}"); }
catch (OperationCanceledException ex) { services.logger.LogDebug($"Websocket task cancelled, this is normal: {ex.Message}"); }
catch (Exception ex) { services.logger.LogError($"WEBSOCKET CRITICAL: UNHANDLED EXCEPTION DURING CANCEL: {ex}"); }
}
if (currentListeners.ContainsKey(trackerId) && !currentListeners.TryRemove(trackerId, out _))
{
services.logger.LogDebug($"Couldn't remove listener {trackerId}, this could be a serious error!");
}
//This won't catch errors in every case but do it anyway
if (socket.State == WebSocketState.Open)
{
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Force closing due to end of task", dualCancel.Token);
}
}
//Just return an empty result if we get all the way to the end. This shouldn't happen but...
return(new EmptyResult());
}
finally
{
//This is SO IMPORTANT that I want to do it way out here!
await RemoveStatusesByTrackerAsync();
}
}
public ExcelSource()
{
Buffer = new BufferBlock <TOutput>();
}
private void SubscribeToConfiguredProject(
ConfiguredProject configuredProject,
IProjectSubscriptionService subscriptionService,
IEnumerable <string> watchedEvaluationRules,
IEnumerable <string> watchedDesignTimeBuildRules)
{
var intermediateBlockDesignTime =
new BufferBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Intermediate DesignTime Input: {1}"
});
var intermediateBlockEvaluation =
new BufferBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Intermediate Evaluation Input: {1}"
});
_subscriptions = _subscriptions ?? new DisposableBag();
_subscriptions.AddDisposable(
subscriptionService.JointRuleSource.SourceBlock.LinkTo(
intermediateBlockDesignTime,
ruleNames: watchedDesignTimeBuildRules.Union(watchedEvaluationRules),
suppressVersionOnlyUpdates: true,
linkOptions: DataflowOption.PropagateCompletion));
_subscriptions.AddDisposable(
subscriptionService.ProjectRuleSource.SourceBlock.LinkTo(
intermediateBlockEvaluation,
ruleNames: watchedEvaluationRules,
suppressVersionOnlyUpdates: true,
linkOptions: DataflowOption.PropagateCompletion));
var actionBlockDesignTimeBuild =
new ActionBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > >(
e => OnProjectChangedAsync(e, configuredProject, RuleHandlerType.DesignTimeBuild),
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget DesignTime Input: {1}"
});
var actionBlockEvaluation =
new ActionBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > >(
e => OnProjectChangedAsync(e, configuredProject, RuleHandlerType.Evaluation),
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Evaluation Input: {1}"
});
_subscriptions.AddDisposable(ProjectDataSources.SyncLinkTo(
intermediateBlockDesignTime.SyncLinkOptions(),
subscriptionService.ProjectCatalogSource.SourceBlock.SyncLinkOptions(),
configuredProject.Capabilities.SourceBlock.SyncLinkOptions(),
actionBlockDesignTimeBuild,
linkOptions: DataflowOption.PropagateCompletion));
_subscriptions.AddDisposable(ProjectDataSources.SyncLinkTo(
intermediateBlockEvaluation.SyncLinkOptions(),
subscriptionService.ProjectCatalogSource.SourceBlock.SyncLinkOptions(),
configuredProject.Capabilities.SourceBlock.SyncLinkOptions(),
actionBlockEvaluation,
linkOptions: DataflowOption.PropagateCompletion));
}
private void SubscribeToConfiguredProject(
ConfiguredProject configuredProject,
IProjectSubscriptionService subscriptionService,
IReadOnlyCollection <string> watchedEvaluationRules,
IReadOnlyCollection <string> watchedDesignTimeBuildRules)
{
// Use intermediate buffer blocks for project rule data to allow subsequent blocks
// to only observe specific rule name(s).
var intermediateBlockDesignTime =
new BufferBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Intermediate DesignTime Input: {1}"
});
var intermediateBlockEvaluation =
new BufferBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Intermediate Evaluation Input: {1}"
});
_subscriptions ??= new DisposableBag();
_subscriptions.AddDisposable(
subscriptionService.JointRuleSource.SourceBlock.LinkTo(
intermediateBlockDesignTime,
ruleNames: watchedDesignTimeBuildRules.Union(watchedEvaluationRules),
suppressVersionOnlyUpdates: true,
linkOptions: DataflowOption.PropagateCompletion));
_subscriptions.AddDisposable(
subscriptionService.ProjectRuleSource.SourceBlock.LinkTo(
intermediateBlockEvaluation,
ruleNames: watchedEvaluationRules,
suppressVersionOnlyUpdates: true,
linkOptions: DataflowOption.PropagateCompletion));
ITargetBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > > actionBlockDesignTimeBuild =
DataflowBlockSlim.CreateActionBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > >(
e => OnProjectChangedAsync(e.Value.Item1, e.Value.Item2, e.Value.Item3, configuredProject, RuleHandlerType.DesignTimeBuild),
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget DesignTime Input: {1}"
});
ITargetBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > > actionBlockEvaluation =
DataflowBlockSlim.CreateActionBlock <IProjectVersionedValue <Tuple <IProjectSubscriptionUpdate, IProjectCatalogSnapshot, IProjectCapabilitiesSnapshot> > >(
e => OnProjectChangedAsync(e.Value.Item1, e.Value.Item2, e.Value.Item3, configuredProject, RuleHandlerType.Evaluation),
new ExecutionDataflowBlockOptions()
{
NameFormat = "CrossTarget Evaluation Input: {1}"
});
_subscriptions.AddDisposable(ProjectDataSources.SyncLinkTo(
intermediateBlockDesignTime.SyncLinkOptions(),
subscriptionService.ProjectCatalogSource.SourceBlock.SyncLinkOptions(),
configuredProject.Capabilities.SourceBlock.SyncLinkOptions(),
actionBlockDesignTimeBuild,
linkOptions: DataflowOption.PropagateCompletion));
_subscriptions.AddDisposable(ProjectDataSources.SyncLinkTo(
intermediateBlockEvaluation.SyncLinkOptions(),
subscriptionService.ProjectCatalogSource.SourceBlock.SyncLinkOptions(),
configuredProject.Capabilities.SourceBlock.SyncLinkOptions(),
actionBlockEvaluation,
linkOptions: DataflowOption.PropagateCompletion));
}
public void TestReadOneLoadingTx()
{
var coreStorageMock = new Mock <ICoreStorage>();
// create a fake transaction with 4 inputs
var prevTxCount = 4;
var txIndex = 1;
var chainedHeader = RandomData.RandomChainedHeader();
// create previous transactions for 4 inputs
var prevTxes = new Transaction[prevTxCount];
var inputs = new TxInput[prevTxCount];
for (var i = 0; i < prevTxCount; i++)
{
var prevTx = RandomData.RandomTransaction();
var prevBlockTx = (BlockTx)BlockTx.Create(i, prevTx);
prevTxes[i] = prevTx;
inputs[i] = new TxInput(prevTx.Hash, 0, ImmutableArray.Create <byte>(), 0);
// mock retrieval of the previous transaction
coreStorageMock.Setup(coreStorage => coreStorage.TryGetTransaction(UInt256.Zero, i, out prevBlockTx)).Returns(true);
}
// create a loading tx with the 4 inputs referencing block hash 0
var tx = RandomData.RandomTransaction(new RandomDataOptions {
TxOutputCount = 1
})
.CreateWith(Inputs: inputs.ToImmutableArray()).Transaction;
var prevOutputTxKeys = ImmutableArray.CreateRange(
Enumerable.Range(0, prevTxCount).Select(x => new TxLookupKey(UInt256.Zero, x)));
var loadingTx = new LoadingTx(txIndex, tx, chainedHeader, prevOutputTxKeys);
// begin queuing transactions to load
var loadingTxes = new BufferBlock <LoadingTx>();
loadingTxes.Post(loadingTx);
loadingTxes.Complete();
// begin transaction loading
var txLoader = TxLoader.LoadTxes(coreStorageMock.Object, loadingTxes);
// verify the loaded transaction
var loadedTxesBuffer = new BufferBlock <LoadedTx>();
txLoader.LinkTo(loadedTxesBuffer, new DataflowLinkOptions {
PropagateCompletion = true
});
txLoader.Completion.Wait();
IList <LoadedTx> actualLoadedTxes;
Assert.IsTrue(loadedTxesBuffer.TryReceiveAll(out actualLoadedTxes));
var actualLoadedTx = actualLoadedTxes.Single();
Assert.AreEqual(loadingTx.TxIndex, actualLoadedTx.TxIndex);
Assert.AreEqual(loadingTx.Transaction, actualLoadedTx.Transaction);
CollectionAssert.AreEqual(prevTxes.Select(x => x.Hash).ToArray(), actualLoadedTx.InputTxes.Select(x => x.Hash).ToArray());
}
internal WorkerLanguageInvoker(ScriptHost host, BindingMetadata trigger, FunctionMetadata functionMetadata,
Collection <FunctionBinding> inputBindings, Collection <FunctionBinding> outputBindings, BufferBlock <ScriptInvocationContext> invocationBuffer)
: base(host, functionMetadata)
{
_trigger = trigger;
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_invocationBuffer = invocationBuffer;
InitializeFileWatcherIfEnabled();
if (_outputBindings.Any(p => p.Metadata.IsReturn))
{
_handleScriptReturnValue = HandleReturnParameter;
}
else
{
_handleScriptReturnValue = HandleOutputDictionary;
}
}
private DataFlowPipeline <ApiLoaderWorkItem> CreateRetryPipeline(int numResourcesToRetry)
{
var totalResources = numResourcesToRetry;
var retryBufferBlock = new BufferBlock <ApiLoaderWorkItem>();
var retryBlock = new TransformBlock <ApiLoaderWorkItem, ApiLoaderWorkItem>(
x => _submitResourcesProcessor.ProcessAsync(x),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = _apiConfiguration.TaskCapacity, MaxDegreeOfParallelism = _apiConfiguration.MaxSimultaneousRequests
});
var successBlock = new TransformBlock <ApiLoaderWorkItem, ApiLoaderWorkItem>(
delegate(ApiLoaderWorkItem resource)
{
_xmlResourceHashCache.Add(resource.Hash);
return(resource);
});
var errorBlock = new TransformBlock <ApiLoaderWorkItem, ApiLoaderWorkItem>(
delegate(ApiLoaderWorkItem resource)
{
LogResource(resource);
SetNonZeroExitCode(resource);
return(resource);
});
var completionCheckBlock = new ActionBlock <ApiLoaderWorkItem>(
delegate
{
totalResources--;
if (totalResources == 0)
{
retryBufferBlock.Complete();
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
retryBufferBlock.LinkTo(
retryBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
}
);
retryBlock.LinkTo(
retryBufferBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
},
x => x.Responses.Count <= _apiConfiguration.Retries && !x.IsSuccess);
retryBlock.LinkTo(
successBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
},
x => x.IsSuccess);
retryBlock.LinkTo(
errorBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
});
successBlock.LinkTo(
completionCheckBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
});
errorBlock.LinkTo(
completionCheckBlock,
new DataflowLinkOptions
{
PropagateCompletion = true
});
return(new DataFlowPipeline <ApiLoaderWorkItem>(retryBufferBlock, completionCheckBlock.Completion));
}
public static void GetHeritageProject(BufferBlock <string> imageTargetBlock)
{
GetMainPageInformation(imageTargetBlock);
GetAllProjectList(imageTargetBlock);
}
public Task <ActionBlock <IList <RootData> > > Process(BufferBlock <IList <string> > batchBlock)
{
throw new NotImplementedException();
}
private static void GetAllProjectList(BufferBlock <string> imageTargetBlock)
{
short errorTime = 0;
var totalPages = DebugHelper.DebugHelperTools.IsDebugMode() ? 2 : 10;
var block = new BufferBlock <string>();
var task = GetHeritageProjectDetailWorker.GenerateProjectDetailPage(block);
for (int i = 1; i < totalPages; i++)
{
if (errorTime > 10)
{
Console.WriteLine("GetAllProjectList: reach the limitation of error time");
break;
}
var currentPage = String.Format(REQUEST_URL, i);
Console.WriteLine("Starting process: " + currentPage);
var requestResult = WebpageHelper.GetRequest(currentPage);
if (string.IsNullOrEmpty(requestResult))
{
errorTime++;
continue;
}
var jsonObject = JsonConvert.DeserializeObject <HeritageProjectRequest>(requestResult);
if (jsonObject.Links.Total_pages != 0 &&
jsonObject.Links.Total_pages != totalPages &&
!DebugHelper.DebugHelperTools.IsDebugMode())
{
totalPages = jsonObject.Links.Total_pages;
}
var list = jsonObject.List;
if (list == null || list.Length == 0)
{
continue;
}
var bsonArray = new List <BsonDocument>();
var heritageType = typeof(HeritageProject);
var properties = typeof(HeritageProject).GetProperties();
for (int j = 0; j < list.Length; j++)
{
var bsonDocument = new BsonDocument();
list[j].Link = "/project_details/" + list[j].Id;
foreach (var property in properties)
{
//反射获取HeritageProject所有属性
//以属性名作为MongoDB存储的Key值
//反射获取对应List当中的值
bsonDocument.Add(property.Name.ToLower(), Regex.Replace(heritageType.GetProperty(property.Name).GetValue(list[j]).ToString(), "<.*?>", string.Empty));
}
if (MongodbChecker.CheckHeritageProjectExist(list[j].Link))
{
Console.WriteLine("Duplicated Heritage Project Link: {0}", list[j].Link);
errorTime++;
continue;
}
block.Post(list[j].Link);
bsonArray.Add(bsonDocument);
}
if (bsonArray.Count != 0)
{
MongodbSaver.SaveHeritageProjectNewsList(bsonArray);
}
if (jsonObject.More != 1)
{
Console.WriteLine("GetAllProjectList: current page is {0}, more equals 0", i);
break;
}
}
block.Complete();
task.Wait();
}
public DataflowJobQueueErrorHandling()
{
_jobs = new BufferBlock <QueueMessage>();
}
public static IDictionary <string, uint> GetTopWordsDataFlow(FileInfo InputFile, char[] Separators, uint TopCount)
{
// Limitations
const int WorkerCount = 12;
var result = new ConcurrentDictionary <string, uint>(StringComparer.InvariantCultureIgnoreCase);
const int BoundedCapacity = 10000;
// Buffer blocks
var bufferBlock = new BufferBlock <string>(
new DataflowBlockOptions {
BoundedCapacity = BoundedCapacity
});
// Split blocks into lines
var splitLineToWordsBlock = new TransformManyBlock <string, string>(
line => line.Split(Separators, StringSplitOptions.RemoveEmptyEntries),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
BoundedCapacity = BoundedCapacity
});
var batchWordsBlock = new BatchBlock <string>(5000);
var trackWordsOccurrencBlock = new ActionBlock <string[]>(words =>
{
// Loop words in lines
foreach (var word in words)
{
// Valid word
if (!TrackWordsClass.IsValidWord(word))
{
continue;
}
// Update word list
result.AddOrUpdate(word, 1, (key, oldVal) => oldVal + 1);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = WorkerCount
});
var defaultLinkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
bufferBlock.LinkTo(splitLineToWordsBlock, defaultLinkOptions);
splitLineToWordsBlock.LinkTo(batchWordsBlock, defaultLinkOptions);
batchWordsBlock.LinkTo(trackWordsOccurrencBlock, defaultLinkOptions);
// Begin producing
foreach (var line in File.ReadLines(InputFile.FullName))
{
bufferBlock.SendAsync(line).Wait();
}
bufferBlock.Complete();
// End of producing
// Wait for workers to finish their work
trackWordsOccurrencBlock.Completion.Wait();
// Return ordered dictionary
return(result
.OrderByDescending(kv => kv.Value)
.Take((int)TopCount)
.ToDictionary(kv => kv.Key, kv => kv.Value));
}
static async Task Main()
{
// <2>
var bufferBlock = new BufferBlock <int>();
// Post several messages to the block.
for (int i = 0; i < 3; i++)
{
bufferBlock.Post(i);
}
// Receive the messages back from the block.
for (int i = 0; i < 3; i++)
{
Console.WriteLine(bufferBlock.Receive());
}
// Output:
// 0
// 1
// 2
// </2>
// <3>
// Post more messages to the block.
for (int i = 0; i < 3; i++)
{
bufferBlock.Post(i);
}
// Receive the messages back from the block.
while (bufferBlock.TryReceive(out int value))
{
Console.WriteLine(value);
}
// Output:
// 0
// 1
// 2
// </3>
// <4>
// Write to and read from the message block concurrently.
var post01 = Task.Run(() =>
{
bufferBlock.Post(0);
bufferBlock.Post(1);
});
var receive = Task.Run(() =>
{
for (int i = 0; i < 3; i++)
{
Console.WriteLine(bufferBlock.Receive());
}
});
var post2 = Task.Run(() =>
{
bufferBlock.Post(2);
});
await Task.WhenAll(post01, receive, post2);
// Output:
// 0
// 1
// 2
// </4>
// Demonstrate asynchronous dataflow operations.
await AsyncSendReceive(bufferBlock);
}
private async Task ConcurrentTellProcess(BufferBlock <DataTaskWrap <IEventBase <K>, bool> > reader)
{
var start = DateTime.UtcNow;
var evtList = new List <IEventBase <K> >();
var startVersion = State.Version;
var startTime = State.VersionTime;
if (UnprocessedList.Count > 0)
{
var startEvt = UnprocessedList.Last();
startVersion = startEvt.Version;
startTime = startEvt.Timestamp;
}
var maxVersion = startVersion;
TaskCompletionSource <bool> maxRequest = default(TaskCompletionSource <bool>);
try
{
while (reader.TryReceiveAll(out var msgs))
{
foreach (var wrap in msgs)
{
if (wrap.Value.Version <= startVersion)
{
wrap.TaskSource.TrySetResult(true);
}
else
{
evtList.Add(wrap.Value);
if (wrap.Value.Version > maxVersion)
{
maxRequest?.TrySetResult(true);
maxVersion = wrap.Value.Version;
maxRequest = wrap.TaskSource;
}
else
{
wrap.TaskSource.TrySetResult(true);
}
}
}
if ((DateTime.UtcNow - start).TotalMilliseconds > MaxDelayOfBatchMilliseconds)
{
break; //保证批量延时不超过200ms
}
}
var orderList = evtList.OrderBy(w => w.Version).ToList();
if (orderList.Count > 0)
{
var inputLast = orderList.Last();
if (startVersion + orderList.Count != inputLast.Version)
{
var eventStorageTask = GetEventStorage();
if (!eventStorageTask.IsCompleted)
{
await eventStorageTask;
}
var loadList = await eventStorageTask.Result.GetListAsync(GrainId, startVersion, inputLast.Version, startTime);
UnprocessedList.AddRange(loadList);
}
else
{
UnprocessedList.AddRange(orderList.Select(w => w));
}
}
if (UnprocessedList.Count > 0)
{
using (var tokenSource = new CancellationTokenSource())
{
var tasks = UnprocessedList.Select(@event =>
{
var task = OnEventDelivered(@event);
if (!task.IsCompleted)
{
return(task.AsTask());
}
else
{
return(Task.CompletedTask);
}
});
var taskOne = Task.WhenAll(tasks);
using (var taskTwo = Task.Delay(EventAsyncProcessTimeoutSeconds, tokenSource.Token))
{
await Task.WhenAny(taskOne, taskTwo);
if (taskOne.Status == TaskStatus.RanToCompletion)
{
tokenSource.Cancel();
var lastEvt = UnprocessedList.Last();
State.UnsafeUpdateVersion(lastEvt.Version, lastEvt.Timestamp);
var saveTask = SaveSnapshotAsync();
if (!saveTask.IsCompleted)
{
await saveTask;
}
UnprocessedList.Clear();
maxRequest?.TrySetResult(true);
}
else
{
maxRequest?.TrySetException(timeoutException);
}
}
}
}
}
catch (Exception e)
{
maxRequest?.TrySetException(e);
}
}
public ActionMessenger(MessengerInfo info) : base(info)
{
m_feedbackBuffer = new BufferBlock <Message>();
m_creator = InstanceFactoryCreator.Instance.CreateFactory <F>();
}
public async Task MainTest()
{
var cts = new CancellationTokenSource();
Task.Run(() =>
{
if (Console.ReadKey().KeyChar == 'c')
{
cts.Cancel();
}
});
var inputBlock = new BufferBlock <int>(new DataflowBlockOptions {
BoundedCapacity = 5, CancellationToken = cts.Token
});
var filter1Block = new TransformBlock <int, decimal>(n =>
{
decimal result = Convert.ToDecimal(n * 0.97);
WriteLine($"Filter 1 send {result} to the next stage on thread id {Thread.CurrentThread.ManagedThreadId}");
Thread.Sleep(TimeSpan.FromMilliseconds(100));
return(result);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
var filter2Block = new TransformBlock <decimal, string>(n =>
{
string result = $"--{n}--";
WriteLine($"Filter 1 send {result} to the next stage on thread id {Thread.CurrentThread.ManagedThreadId}");
Thread.Sleep(TimeSpan.FromMilliseconds(100));
return(result);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
var outputBlock = new ActionBlock <string>
(
s =>
{
WriteLine($"Filter 1 send {s} to the next stage on thread id {Thread.CurrentThread.ManagedThreadId}");
}
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
});
inputBlock.LinkTo(filter1Block, new DataflowLinkOptions {
PropagateCompletion = true
});
filter1Block.LinkTo(filter2Block, new DataflowLinkOptions {
PropagateCompletion = true
});
filter2Block.LinkTo(outputBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
try
{
Parallel.For(0, 20, new ParallelOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cts.Token
}, i =>
{
WriteLine($"added {i} to source data on thread id {Thread.CurrentThread.ManagedThreadId}");
inputBlock.SendAsync(i).GetAwaiter().GetResult();
});
inputBlock.Complete();
await outputBlock.Completion;
WriteLine("Press enter to exist");
}
catch (OperationCanceledException)
{
WriteLine("Operation has been canceled! Press Enter to exit.");
}
}
/// <summary>
/// Pump implementation.
/// </summary>
private async Task RunAsync(Func <Message, Task> processMessage, CancellationToken ct = default(CancellationToken))
{
using (CancellationTokenSource producerCTS = ct.CanBeCanceled
? CancellationTokenSource.CreateLinkedTokenSource(ct)
: new CancellationTokenSource())
{
// This CancellationToken will either be signaled
// externally, or if our consumer errors.
ct = producerCTS.Token;
// Handover between producer and consumer.
DataflowBlockOptions bufferOptions = new DataflowBlockOptions {
// There is no point in dequeuing more messages than we can process,
// so we'll throttle the producer by limiting the buffer capacity.
BoundedCapacity = MaxDegreeOfParallelism,
CancellationToken = ct
};
BufferBlock <Message> buffer = new BufferBlock <Message>(bufferOptions);
Task producer = Task.Run(async() =>
{
try
{
while (_stop.Task.Status != TaskStatus.RanToCompletion)
{
// This line and next line are the *only* two cancellation
// points which will not cause dropped messages.
ct.ThrowIfCancellationRequested();
Task <Message> peekTask = WithCancellation(PeekAsync(MessageQueue), ct);
if (await Task.WhenAny(peekTask, _stop.Task).ConfigureAwait(false) == _stop.Task)
{
// Stop was signaled before PeekAsync returned. Wind down the producer gracefully
// by breaking out and propagating completion to the consumer blocks.
break;
}
await peekTask.ConfigureAwait(false); // Observe Peek exceptions.
ct.ThrowIfCancellationRequested();
// Zero timeout means that we will error if someone else snatches the
// peeked message from the queue before we get to it (due to a race).
// I deemed this better than getting stuck waiting for a message which
// may never arrive, or, worse yet, let this ReceiveAsync run onobserved
// due to a cancellation (if we choose to abandon it like we do PeekAsync).
// You will have to restart the pump if this throws.
// Omit timeout if this behaviour is undesired.
Message message = await ReceiveAsync(MessageQueue, timeout: TimeSpan.Zero).ConfigureAwait(false);
await buffer.SendAsync(message, ct).ConfigureAwait(false);
}
}
finally
{
buffer.Complete();
}
},
ct);
// Wire up the parallel consumers.
ExecutionDataflowBlockOptions executionOptions = new ExecutionDataflowBlockOptions {
CancellationToken = ct,
MaxDegreeOfParallelism = MaxDegreeOfParallelism,
SingleProducerConstrained = true, // We don't require thread safety guarantees.
BoundedCapacity = MaxDegreeOfParallelism,
};
ActionBlock <Message> consumer = new ActionBlock <Message>(async message =>
{
ct.ThrowIfCancellationRequested();
await processMessage(message).ConfigureAwait(false);
},
executionOptions);
buffer.LinkTo(consumer, new DataflowLinkOptions {
PropagateCompletion = true
});
if (await Task.WhenAny(producer, consumer.Completion).ConfigureAwait(false) == consumer.Completion)
{
// If we got here, consumer probably errored. Stop the producer
// before we throw so we don't go dequeuing more messages.
producerCTS.Cancel();
}
// Task.WhenAll checks faulted tasks before checking any
// canceled tasks, so if our consumer threw a legitimate
// execption, that's what will be rethrown, not the OCE.
await Task.WhenAll(producer, consumer.Completion).ConfigureAwait(false);
}
}
protected Projector(
ILogger <TProject> logger,
IStoreEvents events,
TProject process,
int batchSize,
TimeSpan timeout)
{
_logger = logger;
_events = events;
_timer = new Timer(time => {
_batch.TriggerBatch();
});
_queue = new BufferBlock <Envelope>();
_trigger = new TransformBlock <Envelope, Envelope>(
envelope => {
_timer.Change(timeout, Timeout.InfiniteTimeSpan);
return(envelope);
}
);
_batch = new BatchBlock <Envelope>(batchSize);
_process = new TransformBlock <Envelope[], ProjectionResult[]>(async envelopes => {
var time = DateTime.Now;
try {
var results = new ProjectionResult[envelopes.Length];
for (var i = 0; i < envelopes.Length; i++)
{
results[i] = new ProjectionResult {
Envelope = envelopes[i],
Projection = await process.Project(envelopes[i]),
};
}
return(results);
} finally {
_logger.LogDebug($"Processed {envelopes.Length} events in {(DateTime.Now - time).TotalMilliseconds}ms");
}
});
_commit = new ActionBlock <ProjectionResult[]>(async envelopes => {
_timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan);
var time = DateTime.Now;
try {
await Commit(envelopes);
} finally {
_logger.LogInformation($"Committed {envelopes.Length} projections in {(DateTime.Now - time).TotalMilliseconds}ms");
}
});
_queue.LinkTo(_trigger, new DataflowLinkOptions {
PropagateCompletion = true
});
_trigger.LinkTo(_batch, new DataflowLinkOptions {
PropagateCompletion = true
});
_batch.LinkTo(_process, new DataflowLinkOptions {
PropagateCompletion = true
});
_process.LinkTo(_commit, new DataflowLinkOptions {
PropagateCompletion = true
});
}
public async Task <List <NuGetPackage> > DownloadAllPackagesAsync(List <NuGetPackage> packageList)
{
Debug.Assert(packageList != null && packageList.Count > 0);
// urls to download
// TODO: dont hard code the num of producer/consumer
List <NuGetPackage> downloadedPackages1 = new List <NuGetPackage>();
List <NuGetPackage> downloadedPackages2 = new List <NuGetPackage>();
List <NuGetPackage> downloadedPackages3 = new List <NuGetPackage>();
List <NuGetPackage> downloadedPackages4 = new List <NuGetPackage>();
List <NuGetPackage> downloadedPackages5 = new List <NuGetPackage>();
// TODO: rename
BufferBlock <NuGetPackage> urlDownloadQueue = new BufferBlock <NuGetPackage>(new DataflowBlockOptions {
BoundedCapacity = 5
});
var consumerOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 1
};
// todo: needs to be refactored
var consumer1 = new ActionBlock <NuGetPackage>(async pkg => {
var downloadUrl = _nugetHelper.GetDownloadUrlFor(pkg);
var localfilepath = await _remoteFile.GetRemoteFileAsync(downloadUrl, pkg.GetPackageFilename());
pkg.LocalFilepath = localfilepath;
downloadedPackages1.Add(pkg);
}, consumerOptions);
var consumer2 = new ActionBlock <NuGetPackage>(async pkg => {
var downloadUrl = _nugetHelper.GetDownloadUrlFor(pkg);
var localfilepath = await _remoteFile.GetRemoteFileAsync(downloadUrl, pkg.GetPackageFilename());
pkg.LocalFilepath = localfilepath;
downloadedPackages2.Add(pkg);
}, consumerOptions);
var consumer3 = new ActionBlock <NuGetPackage>(async pkg => {
var downloadUrl = _nugetHelper.GetDownloadUrlFor(pkg);
var localfilepath = await _remoteFile.GetRemoteFileAsync(downloadUrl, pkg.GetPackageFilename());
pkg.LocalFilepath = localfilepath;
downloadedPackages3.Add(pkg);
}, consumerOptions);
var consumer4 = new ActionBlock <NuGetPackage>(async pkg => {
var downloadUrl = _nugetHelper.GetDownloadUrlFor(pkg);
var localfilepath = await _remoteFile.GetRemoteFileAsync(downloadUrl, pkg.GetPackageFilename());
pkg.LocalFilepath = localfilepath;
downloadedPackages4.Add(pkg);
}, consumerOptions);
var consumer5 = new ActionBlock <NuGetPackage>(async pkg => {
var downloadUrl = _nugetHelper.GetDownloadUrlFor(pkg);
var localfilepath = await _remoteFile.GetRemoteFileAsync(downloadUrl, pkg.GetPackageFilename());
pkg.LocalFilepath = localfilepath;
downloadedPackages5.Add(pkg);
}, consumerOptions);
urlDownloadQueue.LinkTo(consumer1, new DataflowLinkOptions {
PropagateCompletion = true
});
urlDownloadQueue.LinkTo(consumer2, new DataflowLinkOptions {
PropagateCompletion = true
});
urlDownloadQueue.LinkTo(consumer3, new DataflowLinkOptions {
PropagateCompletion = true
});
urlDownloadQueue.LinkTo(consumer4, new DataflowLinkOptions {
PropagateCompletion = true
});
urlDownloadQueue.LinkTo(consumer5, new DataflowLinkOptions {
PropagateCompletion = true
});
var producer = ProduceAsync(urlDownloadQueue, packageList);
//var consumer = ConsumeAsync(urlDownloadQueue, _remoteFile);
await Task.WhenAll(producer, consumer1.Completion, consumer2.Completion, consumer3.Completion, consumer4.Completion, consumer5.Completion);
var allResults = downloadedPackages1.Concat(downloadedPackages2).Concat(downloadedPackages3).Concat(downloadedPackages4).Concat(downloadedPackages5).ToList();
return(allResults);
}
public DiagnosticPipeline(
IHealthReporter healthReporter,
IReadOnlyCollection <IObservable <EventData> > inputs,
IReadOnlyCollection <IFilter> globalFilters,
IReadOnlyCollection <EventSink> sinks,
DiagnosticPipelineConfiguration pipelineConfiguration = null,
bool disposeDependencies = false)
{
Requires.NotNull(healthReporter, nameof(healthReporter));
Requires.NotNull(inputs, nameof(inputs));
Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input");
Requires.NotNull(sinks, nameof(sinks));
Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink");
this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration();
this.Inputs = inputs;
this.Sinks = sinks;
// Just play nice and make sure there is always something to enumerate on
this.GlobalFilters = globalFilters ?? new IFilter[0];
this.HealthReporter = healthReporter;
this.cancellationTokenSource = new CancellationTokenSource();
var propagateCompletion = new DataflowLinkOptions()
{
PropagateCompletion = true
};
this.pipelineDisposables = new List <IDisposable>();
var inputBuffer = new BufferBlock <EventData>(
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineHead = inputBuffer;
var batcher = new BatchBlock <EventData>(
this.pipelineConfiguration.MaxEventBatchSize,
new GroupingDataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.MaxEventBatchSize,
CancellationToken = this.cancellationTokenSource.Token
}
);
this.pipelineDisposables.Add(inputBuffer.LinkTo(batcher, propagateCompletion));
this.pipelineDisposables.Add(new Timer(
(unused) => batcher.TriggerBatch(),
state: null,
dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec),
period: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec)));
ISourceBlock <EventData[]> sinkSource;
FilterAction filterTransform;
if (this.GlobalFilters.Count > 0)
{
filterTransform = new FilterAction(
this.GlobalFilters,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
var globalFiltersBlock = filterTransform.GetFilterBlock();
this.pipelineDisposables.Add(batcher.LinkTo(globalFiltersBlock, propagateCompletion));
sinkSource = globalFiltersBlock;
}
else
{
sinkSource = batcher;
}
if (sinks.Count > 1)
{
var broadcaster = new BroadcastBlock <EventData[]>(
(events) => events?.Select((e) => e.DeepClone()).ToArray(),
new DataflowBlockOptions()
{
BoundedCapacity = this.pipelineConfiguration.MaxEventBatchSize,
CancellationToken = this.cancellationTokenSource.Token
});
this.pipelineDisposables.Add(sinkSource.LinkTo(broadcaster, propagateCompletion));
sinkSource = broadcaster;
}
foreach (var sink in sinks)
{
ISourceBlock <EventData[]> outputSource = sinkSource;
if (sink.Filters != null && sink.Filters.Count > 0)
{
filterTransform = new FilterAction(
sink.Filters,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
var filterBlock = filterTransform.GetFilterBlock();
this.pipelineDisposables.Add(sinkSource.LinkTo(filterBlock, propagateCompletion));
outputSource = filterBlock;
}
OutputAction outputAction = new OutputAction(
sink.Output,
this.cancellationTokenSource.Token,
this.pipelineConfiguration.MaxEventBatchSize,
this.pipelineConfiguration.MaxConcurrency,
healthReporter);
ActionBlock <EventData[]> outputBlock = outputAction.GetOutputBlock();
this.pipelineDisposables.Add(outputSource.LinkTo(outputBlock, propagateCompletion));
}
IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(inputBuffer, this.HealthReporter);
this.inputSubscriptions = new List <IDisposable>(inputs.Count);
foreach (var input in inputs)
{
this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver));
}
this.disposed = false;
this.disposeDependencies = disposeDependencies;
}
public virtual void LinkTo(TClient client, Action <TCacheEventArgs> callback = null)
{
Throw.IfNull(client, nameof(client));
if (_isLinked)
{
if (client == Client)
{
return; // ignore.
}
throw new InvalidOperationException($"{GetType().Name} is linked to another {Client.GetType().Name}.");
}
Client = client;
_isLinked = true;
_callback = callback;
_bufferBlock = new BufferBlock <TEventArgs>(new DataflowBlockOptions
{
EnsureOrdered = true,
CancellationToken = CancellationToken.None,
BoundedCapacity = DataflowBlockOptions.Unbounded,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded
});
_actionBlock = new ActionBlock <TEventArgs>(async @event =>
{
TCacheEventArgs eventArgs = null;
try
{
eventArgs = await OnAction(@event)
.ConfigureAwait(false);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{GetType().Name}: Unhandled {nameof(OnAction)} exception. [thread: {Thread.CurrentThread.ManagedThreadId}{(@event.Token.IsCancellationRequested ? ", canceled" : string.Empty)}]");
}
if (eventArgs != null)
{
try
{
_callback?.Invoke(eventArgs);
Update?.Invoke(this, eventArgs);
}
catch (OperationCanceledException) { /* ignore */ }
catch (Exception e)
{
Logger?.LogError(e, $"{GetType().Name}: Unhandled update event handler exception. [thread: {Thread.CurrentThread.ManagedThreadId}{(@event.Token.IsCancellationRequested ? ", canceled" : string.Empty)}]");
}
}
}, new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1,
EnsureOrdered = true,
//MaxMessagesPerTask = 1,
MaxDegreeOfParallelism = 1,
CancellationToken = CancellationToken.None,
SingleProducerConstrained = true
});
_bufferBlock.LinkTo(_actionBlock);
}
private static void GetMainPageInformation(BufferBlock <string> imageTargetBlock)
{
var doc = WebpageHelper.GetHttpRequestDocument(PROJECT_MAIN_PAGE);
var nodes = doc.DocumentNode.SelectNodes("//div[@class='x-wrap']/div[@class='title']/div");
if (nodes != null)
{
var bson = new BsonDocument();
foreach (var node in nodes)
{
switch (node.Attributes["class"].Value)
{
case "h30":
bson.Add("title", node.InnerText);
break;
case "p":
StringBuilder sb = new StringBuilder();
foreach (var contentNode in node.ChildNodes)
{
sb.Append(contentNode.InnerText);
}
bson.Add("content", sb.ToString());
break;
}
}
//非遗页面的数字
nodes = doc.DocumentNode.SelectNodes("//div/div[@class='num-item']");
if (nodes != null)
{
var bsonArray = new BsonArray();
foreach (var node in nodes)
{
var numBson = new BsonDocument();
foreach (var childNode in node.ChildNodes)
{
switch (childNode.Attributes["class"]?.Value)
{
case "b":
numBson.Add("num", childNode.Attributes["data-rn"].Value);
break;
case "h18":
numBson.Add("desc", childNode.InnerText);
break;
}
}
bsonArray.Add(numBson);
}
bson.Add("numItem", bsonArray);
}
//获取首页的非物质文化遗产地图
GetHeritageMapTableInformation(bson);
var mongodbBson = MongodbGetter.GetHeritageProjectMainPageDesc();
if (mongodbBson == null)
{
Console.WriteLine("Insert Heritage Project Content");
MongodbSaver.SaveHeritageProjectMainContent(bson);
}
else if (!CheckBsonIsEqual(bson, mongodbBson))
{
Console.WriteLine("Update Heritage Project Content");
MongodbUpdater.UpdateHeritageProjectMainContent(bson);
}
else
{
Console.WriteLine("Not Insert Heritage Project Content");
}
Console.WriteLine(bson);
}
}
