public void Consumer(ISourceBlock<string> source) { var ablock = new ActionBlock<string>( data => this.ParseRawIRC(data), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); source.LinkTo(ablock); }
public static async Task ValidateBlockAsync(ICoreStorage coreStorage, ICoreRules rules, Chain newChain, ISourceBlock<ValidatableTx> validatableTxes, CancellationToken cancelToken = default(CancellationToken)) { // tally transactions object finalTally = null; var txTallier = new TransformBlock<ValidatableTx, ValidatableTx>( validatableTx => { var runningTally = finalTally; rules.TallyTransaction(newChain, validatableTx, ref runningTally); finalTally = runningTally; return validatableTx; }); validatableTxes.LinkTo(txTallier, new DataflowLinkOptions { PropagateCompletion = true }); // validate transactions var txValidator = InitTxValidator(rules, newChain, cancelToken); // begin feeding the tx validator txTallier.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true }); // validate scripts var scriptValidator = InitScriptValidator(rules, newChain, cancelToken); // begin feeding the script validator txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true }); //TODO await PipelineCompletion.Create( new Task[] { }, new IDataflowBlock[] { validatableTxes, txTallier, txValidator, scriptValidator }); // validate overall block rules.PostValidateBlock(newChain, finalTally); }
public void CalculateAverage(ISourceBlock<List<Citizen>> source, Action<int, int> PrintAverage) { if(extractionCompletion == null) { ExtractData(source); } extractionCompletion.ContinueWith((tsk) => PrintAverage(totalAge, peopleCount)); }
public async Task UpdateLinksAsync(AwsManager awsManager, ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> linkBlobs, CancellationToken cancellationToken) { try { await _linkManager.CreateLinksAsync(awsManager, linkBlobs, _s3Settings.ActuallyWrite, cancellationToken).ConfigureAwait(false); Debug.WriteLine("Done processing links"); } catch (Exception ex) { Debug.WriteLine("Processing links failed: " + ex.Message); } }
/// <summary> /// 初期化処理。 /// </summary> /// <param name="settingObject"></param> /// <param name="token"></param> public void Init(dynamic settingObject, CancellationToken token) { logger.Trace("Init Start"); var opt = new DataflowBlockOptions { CancellationToken = token, }; var buffer = new BufferBlock<PastaLog>(opt); var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt); buffer.LinkTo(bloadcast); Target = buffer; Source = bloadcast; logger.Trace("Init End"); }
async Task CreateLinksBlockAsync(AwsManager awsManager, string collection, ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> collectionBlock, bool actuallyWrite, CancellationToken cancellationToken) { var links = await awsManager.GetLinksAsync(collection, cancellationToken).ConfigureAwait(false); Debug.WriteLine($"Link handler for {collection} found {links.Count} existing links"); var createLinkBlock = new ActionBlock<S3Links.ICreateLinkRequest>( link => CreateLinkAsync(awsManager, link, actuallyWrite, cancellationToken), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 512, CancellationToken = cancellationToken }); var makeLinkBlock = new TransformBlock<Tuple<AnnotatedPath, IFileFingerprint>, S3Links.ICreateLinkRequest>( tuple => { var path = tuple.Item1; var file = tuple.Item2; if (collection != path.Collection) throw new InvalidOperationException($"Create link for {path.Collection} on {collection}"); var relativePath = path.RelativePath; if (relativePath.StartsWith("..")) throw new InvalidOperationException($"Create link for invalid path {relativePath}"); if (relativePath.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException($"Create link for invalid path {relativePath}"); relativePath = relativePath.Replace('\\', '/'); if (relativePath.StartsWith("/", StringComparison.Ordinal)) throw new InvalidOperationException($"Create link for invalid path {relativePath}"); string eTag; links.TryGetValue(relativePath, out eTag); return awsManager.BuildLinkRequest(collection, relativePath, file, eTag); }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount }); makeLinkBlock.LinkTo(createLinkBlock, new DataflowLinkOptions { PropagateCompletion = true }, link => null != link); makeLinkBlock.LinkTo(DataflowBlock.NullTarget<S3Links.ICreateLinkRequest>()); collectionBlock.LinkTo(makeLinkBlock, new DataflowLinkOptions { PropagateCompletion = true }); await createLinkBlock.Completion.ConfigureAwait(false); Debug.WriteLine($"Link handler for {collection} is done"); }
public void FiveMostUncommonNames(ISourceBlock<List<Citizen>> source, Action<List<string>, string> PrintResult) { if (extractionCompletion == null) { ExtractData(source); } extractionCompletion.ContinueWith( (arg) => { var sortedDict = from entry in globalNameFrequency orderby entry.Value ascending select entry.Key; var list = sortedDict.Take(5).ToList(); PrintResult(list, "uncommon"); } ); }
public static ISourceBlock<LoadedTx> LoadTxes(ICoreStorage coreStorage, ISourceBlock<LoadingTx> loadingTxes, CancellationToken cancelToken = default(CancellationToken)) { // split incoming LoadingTx by its number of inputs var createTxInputList = InitCreateTxInputList(cancelToken); // link the loading txes to the input splitter loadingTxes.LinkTo(createTxInputList, new DataflowLinkOptions { PropagateCompletion = true }); // load each input, and return and fully loaded txes var loadTxInputAndReturnLoadedTx = InitLoadTxInputAndReturnLoadedTx(coreStorage, cancelToken); // link the input splitter to the input loader createTxInputList.LinkTo(loadTxInputAndReturnLoadedTx, new DataflowLinkOptions { PropagateCompletion = true }); return loadTxInputAndReturnLoadedTx; }
public static ISourceBlock<BlockTx> LookAhead(ISourceBlock<BlockTx> blockTxes, IDeferredChainStateCursor deferredChainStateCursor, CancellationToken cancelToken = default(CancellationToken)) { // capture the original block txes order var orderedBlockTxes = OrderingBlock.CaptureOrder<BlockTx, BlockTx, int>( blockTxes, blockTx => blockTx.Index, cancelToken); // queue each utxo entry to be warmed up: each input's previous transaction, and each new transaction var queueUnspentTxLookup = InitQueueUnspentTxLookup(cancelToken); orderedBlockTxes.LinkTo(queueUnspentTxLookup, new DataflowLinkOptions { PropagateCompletion = true }); // warm up each uxto entry var warmupUtxo = InitWarmupUtxo(deferredChainStateCursor, cancelToken); queueUnspentTxLookup.LinkTo(warmupUtxo, new DataflowLinkOptions { PropagateCompletion = true }); // return the block txes with warmed utxo entries in original order return orderedBlockTxes.ApplyOrder(warmupUtxo, blockTx => blockTx.Index, cancelToken); }
public async Task CreateLinksAsync(AwsManager awsManager, ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> blobSourceBlock, bool actuallyWrite, CancellationToken cancellationToken) { var collectionBlocks = new Dictionary<string, ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>>>(); var tasks = new List<Task>(); var routeBlock = new ActionBlock<Tuple<AnnotatedPath, IFileFingerprint>>(async blob => { var collection = blob.Item1.Collection; if (string.IsNullOrEmpty(collection)) return; ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> collectionBlock; if (!collectionBlocks.TryGetValue(collection, out collectionBlock)) { var bufferBlock = new BufferBlock<Tuple<AnnotatedPath, IFileFingerprint>>(); collectionBlock = bufferBlock; collectionBlocks[collection] = collectionBlock; var task = CreateLinksBlockAsync(awsManager, collection, bufferBlock, actuallyWrite, cancellationToken); tasks.Add(task); } await collectionBlock.SendAsync(blob, cancellationToken).ConfigureAwait(false); }); blobSourceBlock.LinkTo(routeBlock, new DataflowLinkOptions { PropagateCompletion = true }); await routeBlock.Completion.ConfigureAwait(false); Debug.WriteLine("S3LinkCreateor.CreateLinkAsync() routeBlock is done"); foreach (var block in collectionBlocks.Values) block.Complete(); await Task.WhenAll(tasks).ConfigureAwait(false); Debug.WriteLine("S3LinkCreateor.CreateLinkAsync() all link blocks are done"); }
public async Task WorkAsync(ISourceBlock <TEntity> source, CancellationToken c) { var bufferSize = _options .Value.BufferSize.GetOrDefault(typeof(TEntity).Name, _options.Value.DefaultBufferSize); var buffer = new TEntity[bufferSize]; _logger.LogInformation("Start working, buffer size: {0}", bufferSize); while (await source.OutputAvailableAsync(c)) { var i = 0; for (; i < bufferSize && await source.OutputAvailableAsync(c); i++) { buffer[i] = await source.ReceiveAsync(c); } await Insert(buffer.Take(i)); } }
/// <summary> /// Pipes the source data through a single producer constrained ActionBlock. /// </summary> /// <typeparam name="T">The input type.</typeparam> /// <param name="source">The source block to receive from.</param> /// <param name="handler">The handler function to apply.</param> /// <param name="options">Optional execution options.</param> /// <returns>The ActionBlock created.</returns> public static ActionBlock <T> Pipe <T>(this ISourceBlock <T> source, Action <T> handler, ExecutionDataflowBlockOptions?options = null) { if (source is null) { throw new NullReferenceException(); } if (handler is null) { throw new ArgumentNullException(nameof(handler)); } Contract.EndContractBlock(); var receiver = options is null ? new ActionBlock <T>(handler) : new ActionBlock <T>(handler, options); source.LinkToWithCompletion(receiver); return(receiver); }
public ServiceDiscovery(string serviceName, ReachabilityChecker reachabilityChecker, IRemoteHostPoolFactory remoteHostPoolFactory, IDiscoverySourceLoader serviceDiscoveryLoader, IEnvironment environment, ISourceBlock <DiscoveryConfig> configListener, Func <DiscoveryConfig> discoveryConfigFactory, ILog log) { Log = log; _serviceName = serviceName; _originatingDeployment = new DeploymentIdentifier(serviceName, environment.DeploymentEnvironment, environment); _masterDeployment = new DeploymentIdentifier(serviceName, MASTER_ENVIRONMENT, environment); _reachabilityChecker = reachabilityChecker; _remoteHostPoolFactory = remoteHostPoolFactory; _serviceDiscoveryLoader = serviceDiscoveryLoader; _initTask = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory())); _configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost)); }
public BasePingParser(Func <string, Ping> parse) { _source = new BufferBlock <Ping>(); _target = new ActionBlock <string>(async msg => await((BufferBlock <Ping>)_source).SendAsync(parse(msg)), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); _target.Completion.ContinueWith((t) => { if (t.IsFaulted) { _source.Fault(t.Exception); } else { _source.Complete(); } }); }
public static async void ConsumeMediaAsync(string path, ISourceBlock <KeyValuePair <string, string> > source) { if (!File.Exists(path)) { Directory.CreateDirectory(path); } var filesProcessed = 0; var filesDownloaded = 0; while (await source.OutputAvailableAsync()) { var client = new WebClient(); var(fileName, fileUri) = source.Receive(); filesProcessed++; //Conflicted about whether or not this should exist. Doesn't save time, but can fix (or break) //previous downloads. Logger.Info("ConsumeMediaAsync|" + filesProcessed + " Processing: " + fileName); var fileExists = Directory.GetFiles(path, fileName + ".*"); if (fileExists.Length > 0) { continue; } Logger.Info("ConsumeMediaAsync|" + filesProcessed + " Downloading: " + fileName); if (fileUri.Contains(".mp4")) { client.DownloadFileAsync(new Uri(fileUri), path + fileName + ".mp4"); } else { client.DownloadFileAsync(new Uri(fileUri), path + fileName + ".jpg"); } filesDownloaded++; } Logger.Info("ConsumeMediaAsync|" + "Processed {0} files.", filesProcessed); Logger.Info("ConsumeMediaAsync|" + "Downloaded {0} files.", filesDownloaded); }
public async Task <int> SaveFileAsync(ISourceBlock <string> source) { while (await source.OutputAvailableAsync()) { WebClient wc = new WebClient(); if (!Directory.Exists("img")) { Directory.CreateDirectory("img"); } var imageUrl = source.Receive(); if (!imageUrl.StartsWith(GetIhChina.MainPage)) { imageUrl = GetIhChina.MainPage + imageUrl; } var savePath = Path.Combine(Directory.GetCurrentDirectory(), "img", WebpageHelper.GetSubUrl(imageUrl)); if (File.Exists(savePath)) { continue; } try { Console.WriteLine("Starting to save image {0}", imageUrl); wc.DownloadFile(imageUrl, savePath); //CompressImageWithGuetzli(savePath); 压缩貌似有点问题,待修复 CorpImage(savePath); Console.WriteLine("Save image {0} completely", imageUrl); } catch (WebException e) { Console.WriteLine("Save image {0} error", imageUrl); Console.WriteLine(e); } catch (Exception e) { Console.WriteLine("unhandled error"); Console.WriteLine(e); } } return(1); }
public static async Task ValidateBlockAsync(ICoreStorage coreStorage, IBlockchainRules rules, ChainedHeader chainedHeader, ISourceBlock<LoadedTx> loadedTxes, CancellationToken cancelToken = default(CancellationToken)) { // validate merkle root var merkleStream = new MerkleStream(); var merkleValidator = InitMerkleValidator(chainedHeader, merkleStream, cancelToken); // begin feeding the merkle validator loadedTxes.LinkTo(merkleValidator, new DataflowLinkOptions { PropagateCompletion = true }); // validate transactions var txValidator = InitTxValidator(rules, chainedHeader, cancelToken); // begin feeding the tx validator merkleValidator.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true }); // validate scripts var scriptValidator = InitScriptValidator(rules, chainedHeader, cancelToken); // begin feeding the script validator txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true }); await merkleValidator.Completion; await txValidator.Completion; await scriptValidator.Completion; if (!rules.BypassPrevTxLoading) { try { merkleStream.FinishPairing(); } //TODO catch (InvalidOperationException) { throw CreateMerkleRootException(chainedHeader); } if (merkleStream.RootNode.Hash != chainedHeader.MerkleRoot) throw CreateMerkleRootException(chainedHeader); } }
public ConsumerWorkflow <TState> BuildLinkages(DataflowLinkOptions overrideOptions = null) { Guard.AgainstNull(_deserializeBlock, nameof(_deserializeBlock)); Guard.AgainstNull(_errorBuffer, nameof(_errorBuffer)); Guard.AgainstNull(_finalization, nameof(_finalization)); if (_inputBuffer == null) { _inputBuffer = new BufferBlock <ReceivedData>(); } if (_readyBuffer == null) { _readyBuffer = new BufferBlock <TState>(); } if (_postProcessingBuffer == null) { _postProcessingBuffer = new BufferBlock <TState>(); } for (int i = 0; i < ConsumerCount; i++) { _consumerBlocks.Add(new ConsumerBlock <ReceivedData>(_consumer)); _consumerBlocks[i].LinkTo(_inputBuffer, overrideOptions ?? _linkStepOptions); } _inputBuffer.LinkTo(_deserializeBlock, overrideOptions ?? _linkStepOptions); _deserializeBlock.LinkTo(_errorBuffer, overrideOptions ?? _linkStepOptions, x => x == null); SetCurrentSourceBlock(_deserializeBlock); LinkPreProcessing(overrideOptions); LinkSuppliedSteps(overrideOptions); LinkPostProcessing(overrideOptions); _errorBuffer.LinkTo(_errorAction, overrideOptions ?? _linkStepOptions); _currentBlock = null; return(this); }
public EnumerableSourceCore(ISourceBlock <T> parent, Action enumerate, TaskScheduler?taskScheduler, CancellationToken cancellationToken) { this._parent = parent; this._cancellationToken = cancellationToken; if (taskScheduler == null || taskScheduler == TaskScheduler.Default) { #if THREADPOOL WaitCallback enumerateCb = x => ((Action)x).Invoke(); this._enumerate = () => ThreadPool.QueueUserWorkItem(enumerateCb, enumerate); WaitCallback offerCb = x => ((EnumerableSourceCore <T>)x).OfferToTargets(); this._offerToTargetsOnTaskScheduler = () => ThreadPool.QueueUserWorkItem(offerCb, this); #else this._enumerate = () => Task.Run(enumerate); Action offerAction = this.OfferToTargets; this._offerToTargetsOnTaskScheduler = () => Task.Run(offerAction); #endif } else { var taskFactory = new TaskFactory(taskScheduler); this._enumerate = () => taskFactory.StartNew(enumerate); Action offerAction = this.OfferToTargets; this._offerToTargetsOnTaskScheduler = () => taskFactory.StartNew(offerAction); } if (cancellationToken.IsCancellationRequested) { this.State = StateEnum.Completed; this._completeRequested = true; this._tcs.TrySetCanceled(cancellationToken); } else if (cancellationToken.CanBeCanceled) { this._cancelReg = cancellationToken.Register(() => this.Complete(false)); } }
public ISourceBlock <TCombine> ApplyOrder(ISourceBlock <TCombine> source, Func <TCombine, TKey> keyFunc, CancellationToken cancelToken = default(CancellationToken)) { if (sorted) { throw new InvalidOperationException($"{GetType().Name} has already been sorted."); } // sort items back into original order var pendingItems = new Dictionary <TKey, TCombine>(); var sorter = new TransformManyBlock <TCombine, TCombine>( item => { pendingItems.Add(keyFunc(item), item); var sortedItems = new List <TCombine>(); // look to see if the next item in original order has been loaded // if so, dequeue and return the original item and then continue looking for the next in order TKey nextKey; TCombine nextItem; while (orderedKeys.TryPeek(out nextKey) && pendingItems.TryGetValue(nextKey, out nextItem)) { sortedItems.Add(nextItem); pendingItems.Remove(nextKey); orderedKeys.TryDequeue(out nextKey); } return(sortedItems); }, new ExecutionDataflowBlockOptions { CancellationToken = cancelToken }); source.LinkTo(sorter, new DataflowLinkOptions { PropagateCompletion = true }); sorted = true; return(sorter); }
private async Task BlockKeepsPostponedMessages(Func <ISourceBlock <int> > BlockFactory) { ISourceBlock <int> block = BlockFactory(); ITargetBlock <int> blockT = (ITargetBlock <int>)block; TestTargetBlock <int> testTarget = new TestTargetBlock <int>(); testTarget.ConsumptionMode = DataflowMessageStatus.Postponed; block.LinkTo(testTarget, PropagateCompletion); // Assumption: if the target of the block is postponing messages, // The block will accept incoming messages until it runs out of capacity. Assert.True(blockT.Post(1)); Assert.True(blockT.Post(2)); Assert.True(blockT.Post(3)); // Out of capacity Assert.False(blockT.Post(4)); // However SendAsync() will allow postponing the message, sot the message will be eventually delivered blockT.SendAsync(5).Forget(); // Wait till the block offers a message // Assumption: only one message will be offered, the block will not offer more messages if the target postpones bool messageOffered = await TaskUtils.PollWaitAsync(() => testTarget.MessagesPostponed.Count == 1, MessageArrivalTimeout); Assert.True(messageOffered); // Assumption: once the block target stops postponing, the block will keep pushing data to target // until it runs out of buffered messages. testTarget.ConsumptionMode = DataflowMessageStatus.Accepted; testTarget.ConsumePostponedMessages(); // Use 10 times the normal message arrival timeout for extra padding--the test tended to be a bit flakey at this point. // If this happens again, switch to custom TaskScheduler. bool gotAllMessages = await TaskUtils.PollWaitAsync(() => testTarget.MessagesConsumed.Count == 4, TimeSpan.FromMilliseconds(MessageArrivalTimeout.TotalMilliseconds * 10)); Assert.True(gotAllMessages, "We should have gotten 4 messages"); Assert.Equal(testTarget.MessagesConsumed.OrderBy((i) => i), new int[] { 1, 2, 3, 5 }); }
/// <summary> /// Receives an envelope from the buffer. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="buffer">The buffer.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns></returns> /// <exception cref="System.InvalidOperationException"></exception> private async Task <T> ReceiveFromBufferAsync <T>(ISourceBlock <T> buffer, CancellationToken cancellationToken) where T : Envelope, new() { if (State < SessionState.Established) { throw new InvalidOperationException($"Cannot receive envelopes in the '{State}' session state"); } try { return(await buffer.ReceiveAsync(cancellationToken).ConfigureAwait(false)); } catch (InvalidOperationException ex) when(buffer.Completion.IsCompleted) { if (_consumeTransportTask != null) { await _consumeTransportTask.ConfigureAwait(false); _consumeTransportTask = null; } throw new InvalidOperationException("The channel listener task is complete and cannot receive envelopes", ex); } }
public DataflowMessageStatus OfferMessage( DataflowMessageHeader messageHeader, TOutput messageValue, ISourceBlock <TOutput> source, bool consumeToAccept) { if (!messageHeader.IsValid) { return(DataflowMessageStatus.Declined); } if (completion.Task.Status != TaskStatus.WaitingForActivation) { return(DataflowMessageStatus.DecliningPermanently); } lock (completion) { if (completion.Task.Status != TaskStatus.WaitingForActivation) { return(DataflowMessageStatus.DecliningPermanently); } if (consumeToAccept) { bool consummed; if (!source.ReserveMessage(messageHeader, this)) { return(DataflowMessageStatus.NotAvailable); } messageValue = source.ConsumeMessage(messageHeader, this, out consummed); if (!consummed) { return(DataflowMessageStatus.NotAvailable); } } completion.TrySetResult(messageValue); } CompletionSet(); return(DataflowMessageStatus.Accepted); }
public DataflowMessageStatus OfferMessage( DataflowMessageHeader messageHeader, TSource messageValue, ISourceBlock <TSource> source, bool consumeToAccept) { if (consumeToAccept) { if (!source.ReserveMessage(messageHeader, this)) { return(DataflowMessageStatus.NotAvailable); } bool consumed; messageValue = source.ConsumeMessage(messageHeader, this, out consumed); if (!consumed) { return(DataflowMessageStatus.NotAvailable); } } observer.OnNext(messageValue); return(DataflowMessageStatus.Accepted); }
public static Task <int> Choose <T1, T2> (ISourceBlock <T1> source1, Action <T1> action1, ISourceBlock <T2> source2, Action <T2> action2, DataflowBlockOptions dataflowBlockOptions) { if (source1 == null) { throw new ArgumentNullException("source1"); } if (source2 == null) { throw new ArgumentNullException("source2"); } var chooser = new ChooserBlock <T1, T2, object> (action1, action2, null, dataflowBlockOptions); source1.LinkTo(chooser.Target1); source2.LinkTo(chooser.Target2); return(chooser.Completion); }
public ServiceDiscovery(string serviceName, ReachabilityChecker reachabilityChecker, IRemoteHostPoolFactory remoteHostPoolFactory, IDiscoverySourceLoader serviceDiscoveryLoader, IEnvironmentVariableProvider environmentVariableProvider, ISourceBlock <DiscoveryConfig> configListener, Func <DiscoveryConfig> discoveryConfigFactory, ILog log) { Log = log; _serviceName = serviceName; _originatingDeployment = new ServiceDeployment(serviceName, environmentVariableProvider.DeploymentEnvironment); _masterDeployment = new ServiceDeployment(serviceName, MASTER_ENVIRONMENT); _reachabilityChecker = reachabilityChecker; _remoteHostPoolFactory = remoteHostPoolFactory; _serviceDiscoveryLoader = serviceDiscoveryLoader; // Must be run in Task.Run() because of incorrect Orleans scheduling _initTask = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory())); _configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost)); }
public DataflowMessageStatus OfferMessage( DataflowMessageHeader messageHeader, TMessage messageValue, ISourceBlock <TMessage> source, bool consumeToAccept) { if (!chooserBlock.canAccept) { return(DataflowMessageStatus.DecliningPermanently); } bool lockTaken = false; try { chooserBlock.messageLock.Enter(ref lockTaken); if (!chooserBlock.canAccept) { return(DataflowMessageStatus.DecliningPermanently); } if (consumeToAccept) { bool consummed; messageValue = source.ConsumeMessage(messageHeader, this, out consummed); if (!consummed) { return(DataflowMessageStatus.NotAvailable); } } chooserBlock.canAccept = false; } finally { if (lockTaken) { chooserBlock.messageLock.Exit(); } } chooserBlock.MessageArrived(index, action, messageValue); return(DataflowMessageStatus.Accepted); }
public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, TOutput messageValue, ISourceBlock <TOutput> source, bool consumeToAccept) { if (!messageHeader.IsValid) { return(DataflowMessageStatus.Declined); } if (consumeToAccept) { bool consummed; if (!source.ReserveMessage(messageHeader, this)) { return(DataflowMessageStatus.NotAvailable); } messageValue = source.ConsumeMessage(messageHeader, this, out consummed); if (!consummed) { return(DataflowMessageStatus.NotAvailable); } } ReceivedValue = messageValue; completion.TrySetResult(messageValue); Thread.MemoryBarrier(); waitHandle.Set(); /* We do the unlinking here so that we don't get called twice */ if (linkBridge != null) { linkBridge.Dispose(); linkBridge = null; } return(DataflowMessageStatus.Accepted); }
public static bool TryReceiveItem <T>(this ISourceBlock <T> block, out T value) { if (block is IReceivableSourceBlock <T> receiveSourceBlock) { return(receiveSourceBlock.TryReceive(out value)); } try { value = block.Receive(TimeSpan.Zero); return(true); }catch (TimeoutException) { value = default; return(false); } catch (InvalidOperationException) { value = default; return(false); } }
public DataflowMessageStatus OfferMessage( DataflowMessageHeader messageHeader, T messageValue, ISourceBlock <T> source, bool consumeToAccept) { if (!IsOpen) { return(DataflowMessageStatus.Declined); } if (consumeToAccept) { source.ConsumeMessage(messageHeader, this, out var messageConsumed); if (!messageConsumed) { return(DataflowMessageStatus.Declined); } } _toPropergate = messageValue; _links.ForEach(target => target.OfferMessage(messageHeader, messageValue, this, true)); return(DataflowMessageStatus.Accepted); }
public static OrderedSource <TInput, TOutput, TKey> CaptureOrder <TInput, TOutput, TKey>( ISourceBlock <TInput> source, Func <TInput, TKey> keyFunc, CancellationToken cancelToken = default(CancellationToken)) { // capture the original order to be reapplied later var orderedKeys = new ConcurrentQueue <TKey>(); var orderCapturer = new TransformBlock <TInput, TInput>( item => { orderedKeys.Enqueue(keyFunc(item)); return(item); }, new ExecutionDataflowBlockOptions { CancellationToken = cancelToken }); source.LinkTo(orderCapturer, new DataflowLinkOptions { PropagateCompletion = true }); return(new OrderedSource <TInput, TOutput, TKey>(orderedKeys, orderCapturer)); }
static void SourceBlockTestInternal <T>(ISourceBlock <T> block) { var target = new BufferBlock <T> (); bool consumed; // invalid header AssertEx.Throws <ArgumentException> ( () => block.ConsumeMessage(new DataflowMessageHeader(), target, out consumed)); // header that wasn't sent by the block doesn't throw block.ConsumeMessage(new DataflowMessageHeader(1), target, out consumed); AssertEx.Throws <ArgumentNullException> ( () => block.ConsumeMessage(new DataflowMessageHeader(1), null, out consumed)); AssertEx.Throws <ArgumentException> ( () => block.ReserveMessage(new DataflowMessageHeader(), target)); // header that wasn't sent by the block doesn't throw block.ReserveMessage(new DataflowMessageHeader(1), target); AssertEx.Throws <ArgumentNullException> ( () => block.ReserveMessage(new DataflowMessageHeader(1), null)); AssertEx.Throws <ArgumentException> ( () => block.ReleaseReservation(new DataflowMessageHeader(), target)); AssertEx.Throws <ArgumentNullException>(() => block.LinkTo(null, new DataflowLinkOptions())); AssertEx.Throws <ArgumentNullException>(() => block.LinkTo(target, null)); }
//Linkd MY block to OHTER dataflow protected void LinkBlockToFlow <T>(ISourceBlock <T> block, IDataflow <T> otherDataflow, Predicate <T> predicate = null) { if (!IsMyChild(block)) { throw new InvalidOperationException(string.Format("{0} Cannot link block to flow as the output block is not my child.", this.FullName)); } if (predicate == null) { block.LinkTo(otherDataflow.InputBlock); } else { block.LinkTo(otherDataflow.InputBlock, predicate); } otherDataflow.RegisterDependency(this); //Make sure other dataflow also fails me otherDataflow.Completion.ContinueWith(otherTask => { if (this.Completion.IsCompleted) { return; } if (otherTask.IsFaulted) { LogHelper.Logger.InfoFormat("{0} Downstream dataflow faulted before I am done. Fault myself.", this.FullName); this.Fault(new LinkedDataflowFailedException()); } else if (otherTask.IsCanceled) { LogHelper.Logger.InfoFormat("{0} Downstream dataflow canceled before I am done. Cancel myself.", this.FullName); this.Fault(new LinkedDataflowCanceledException()); } }); }
public static ISourceBlock <DecodedBlockTx> LookAhead(ISourceBlock <DecodedBlockTx> blockTxes, IDeferredChainStateCursor deferredChainStateCursor, CancellationToken cancelToken = default(CancellationToken)) { // capture the original block txes order var orderedBlockTxes = OrderingBlock.CaptureOrder <DecodedBlockTx, DecodedBlockTx, int>( blockTxes, blockTx => blockTx.Index, cancelToken); // queue each utxo entry to be warmed up: each input's previous transaction, and each new transaction var queueUnspentTxLookup = InitQueueUnspentTxLookup(cancelToken); orderedBlockTxes.LinkTo(queueUnspentTxLookup, new DataflowLinkOptions { PropagateCompletion = true }); // warm up each uxto entry var warmupUtxo = InitWarmupUtxo(deferredChainStateCursor, cancelToken); queueUnspentTxLookup.LinkTo(warmupUtxo, new DataflowLinkOptions { PropagateCompletion = true }); // return the block txes with warmed utxo entries in original order return(orderedBlockTxes.ApplyOrder(warmupUtxo, blockTx => blockTx.Index, cancelToken)); }
public async Task SerializeToJson(ISourceBlock <Person> source) { using (FileStream fs = File.Create(JsonPAth)) using (StreamWriter sw = new StreamWriter(fs)) using (JsonWriter jw = new JsonTextWriter(sw)) { jw.Formatting = Formatting.Indented; jw.WriteStartObject(); jw.WritePropertyName("People"); jw.WriteStartArray(); string temp = string.Empty; while (await source.OutputAvailableAsync()) { var person = source.Receive(); temp = JsonConvert.SerializeObject(person); jw.WriteValue(temp); } jw.WriteEndArray(); jw.WriteEndObject(); //serializer.Serialize(jw, People); } }
internal static async Task TestReserveAndConsume <T>( ISourceBlock <T> block, bool reservationIsTargetSpecific = true) { bool consumed; block.ConsumeMessage(new DataflowMessageHeader(-99), new ActionBlock <T>(i => { }), out consumed); Assert.False(consumed); var tcs = new TaskCompletionSource <bool>(); var offeredMessage = default(DataflowMessageHeader); var target = new DelegatePropagator <T, T> { OfferMessageDelegate = (messageHeader, value, source, consumeToAccept) => { offeredMessage = messageHeader; tcs.TrySetResult(true); return(DataflowMessageStatus.Postponed); } }; block.LinkTo(target); await tcs.Task; Assert.True(block.ReserveMessage(offeredMessage, target)); // reserve the message if (reservationIsTargetSpecific) { block.ConsumeMessage(offeredMessage, new ActionBlock <T>(delegate { }), out consumed); // different target tries to consume Assert.False(consumed); } block.ConsumeMessage(new DataflowMessageHeader(-99), target, out consumed); // right target, wrong message Assert.False(consumed); block.ConsumeMessage(offeredMessage, target, out consumed); // right target, right message Assert.True(consumed); }
private async Task StuckCompletionCannotBeCancelled(Func <CancellationToken, ISourceBlock <int> > BlockFactory, Func <ISourceBlock <int>, int> OutputCount) { CancellationTokenSource cts = new CancellationTokenSource(); ISourceBlock <int> block = BlockFactory(cts.Token); ITargetBlock <int> blockT = (ITargetBlock <int>)block; TestTargetBlock <int> testTarget = new TestTargetBlock <int>(); testTarget.ConsumptionMode = DataflowMessageStatus.Declined; block.LinkTo(testTarget, PropagateCompletion); Assert.True(blockT.Post(1)); Assert.True(blockT.Post(2)); block.Complete(); // Completion task should still be running await Task.WhenAny(block.Completion, Task.Delay(CompletionTimeout)); Assert.False(block.Completion.IsCompleted); // Assumption: BufferBlock will not start target's completion until it itself completes await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout)); Assert.True(testTarget.Completion.IsNotStarted()); // Assumption: cancellation does not affect the completion of the block (unfortunately!) cts.Cancel(); await Task.WhenAny(block.Completion, Task.Delay(CompletionTimeout)); Assert.False(block.Completion.IsCompleted); await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout)); Assert.True(testTarget.Completion.IsNotStarted()); Assert.Equal(2, OutputCount(block)); }
public static ISourceBlock <Tuple <TOut, TOut> > ForkJoinThen <TIn, TOut> ( this ISourceBlock <TIn> source, Func <ISourceBlock <TIn>, ISourceBlock <TOut> > left, Func <ISourceBlock <TIn>, ISourceBlock <TOut> > right ) { var fork = new BroadcastBlock <TIn>(data => data, new DataflowBlockOptions { NameFormat = "Fork {1}" }); source.LinkTo(fork, LinkOptions); var join = new JoinBlock <TOut, TOut>(); left(fork).LinkTo(join.Target1, LinkOptions); right(fork).LinkTo(join.Target2, LinkOptions); return(join); }
public ConsulClient(string serviceName, Func <ConsulConfig> getConfig, ISourceBlock <ConsulConfig> configChanged, IEnvironmentVariableProvider environmentVariableProvider, ILog log, IDateTime dateTime, Func <string, AggregatingHealthStatus> getAggregatedHealthStatus) { _serviceName = serviceName; _serviceNameOrigin = serviceName; GetConfig = getConfig; _dateTime = dateTime; Log = log; DataCenter = environmentVariableProvider.DataCenter; _waitForConfigChange = new TaskCompletionSource <bool>(); configChanged.LinkTo(new ActionBlock <ConsulConfig>(ConfigChanged)); var address = environmentVariableProvider.ConsulAddress ?? $"{CurrentApplicationInfo.HostName}:8500"; ConsulAddress = new Uri($"http://{address}"); _aggregatedHealthStatus = getAggregatedHealthStatus("ConsulClient"); _resultChanged = new BufferBlock <EndPointsResult>(); _initializedVersion = new TaskCompletionSource <bool>(); ShutdownToken = new CancellationTokenSource(); }
public static async Task <int> GenerateProjectDetailPage(ISourceBlock <string> urlSource) { var errTime = 0; while (await urlSource.OutputAvailableAsync()) { if (errTime == 10) { return(DUPLICATED_NEWS); } var url = urlSource.Receive(); if (MongodbChecker.CheckHeritageProjectDetailExist(url)) { errTime++; continue; } var bson = GenerateHeritageProjectDetailPage(url, true); if (bson != null) { MongodbSaver.SaveHeritageProjectDetail(bson); } } return(PROCESS_SUCCESS); }
private static async Task<int> processBufferBlock(ISourceBlock<string> urlSource, Checker checker, Saver saver, BufferBlock<string> block) { var errorTime = 0; while (await urlSource.OutputAvailableAsync()) { if(errorTime==10) { return DUPLICATED_NEWS; } var url = urlSource.Receive(); if (checker(url)) { Console.WriteLine("duplicated url: {0}", url); errorTime++; continue; } var bson = processDetailPage(url, block); if (bson != null) { saver(bson); } } return PROCESS_SUCCESS; ; }
public void TopThreeMonth(ISourceBlock<List<Citizen>> source, Action<List<string>, string> PrintResult) { if (extractionCompletion == null) { ExtractData(source); } extractionCompletion.ContinueWith( (arg) => { var sortedDict = from entry in globalMonthFrequency orderby entry.Value ascending select entry.Key; var list = sortedDict.Take(3).ToList(); List<string> months = new List<string>(); foreach(int month in list) { months.Add(CultureInfo.CurrentCulture.DateTimeFormat.GetMonthName(month)); } PrintResult(months, "month"); } ); }
public Task GenerateFileFingerprintsAsync(ISourceBlock<AnnotatedPath[]> annotatedPathSourceBlock, ITargetBlock<IFileFingerprint> fileFingerprintTargetBlock, CancellationToken cancellationToken) { var bufferBlock = new BufferBlock<IFileFingerprint>(new DataflowBlockOptions { CancellationToken = cancellationToken }); bufferBlock.LinkTo(fileFingerprintTargetBlock, new DataflowLinkOptions { PropagateCompletion = true }); var storeBatchBlock = new BatchBlock<IFileFingerprint>(FlushCount, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken }); var broadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken }); broadcastBlock.LinkTo(storeBatchBlock, new DataflowLinkOptions { PropagateCompletion = true }, ff => !ff.WasCached); broadcastBlock.LinkTo(bufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); #if DEBUG var task = storeBatchBlock.Completion .ContinueWith(_ => Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() storeBatchBlock completed")); TaskCollector.Default.Add(task, "FileFingerprintManager.GenerateFileFingerprintsAsync storeBatchBlock"); task = bufferBlock.Completion .ContinueWith(_ => Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() bufferBlock completed")); TaskCollector.Default.Add(task, "FileFingerprintManager.GenerateFileFingerprintsAsync bufferBlock"); #endif // DEBUG var storeTask = StoreFileFingerprintsAsync(storeBatchBlock, cancellationToken); var transformTask = TransformAnnotatedPathsToFileFingerprint(annotatedPathSourceBlock, broadcastBlock, cancellationToken); return Task.WhenAll(storeTask, transformTask); }
public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket bucket, ISourceBlock<Bucket> source, bool consumeToAccept) { Console.WriteLine(bucket.ToString()); return DataflowMessageStatus.Accepted; }
Task StoreFileFingerprintsAsync(ISourceBlock<IFileFingerprint[]> storeBatchBlock, CancellationToken cancellationToken) { var block = new ActionBlock<IFileFingerprint[]>( fileFingerprints => WriteBlobsAsync(fileFingerprints, cancellationToken), new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken }); storeBatchBlock.LinkTo(block, new DataflowLinkOptions { PropagateCompletion = true }); return block.Completion; }
private void ExtractData(ISourceBlock<List<Citizen>> source) { var extractData = new ActionBlock<List<Citizen>>( (citizens) => { var today = DateTime.Today; Dictionary<string, int> localNameFrequency = new Dictionary<string, int>(); Dictionary<int, int> localMonthFrequency = new Dictionary<int, int>(); foreach (Citizen citizen in citizens) { var bday = citizen.Birthday; int age = today.Year - bday.Year; if (bday > today.AddYears(-age)) { age--; } Interlocked.Add(ref totalAge, age); Interlocked.Increment(ref peopleCount); string name = citizen.Firstname; if (!localNameFrequency.ContainsKey(name)) { localNameFrequency[name] = 0; } localNameFrequency[name]++; int month = citizen.Birthday.Month; if (!localMonthFrequency.ContainsKey(month)) { localMonthFrequency[month] = 0; } localMonthFrequency[month]++; } mutex.WaitOne(); foreach (string key in localNameFrequency.Keys) { if (!globalNameFrequency.ContainsKey(key)) { globalNameFrequency[key] = 0; } globalNameFrequency[key] += localNameFrequency[key]; } mutex.ReleaseMutex(); mutex.WaitOne(); foreach (int key in localMonthFrequency.Keys) { if (!globalMonthFrequency.ContainsKey(key)) { globalMonthFrequency[key] = 0; } globalMonthFrequency[key] += localMonthFrequency[key]; } mutex.ReleaseMutex(); } , new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount } ); source.LinkTo(extractData, new DataflowLinkOptions { PropagateCompletion = true }); extractionCompletion = extractData.Completion; }
async Task TransformAnnotatedPathsToFileFingerprint(ISourceBlock<AnnotatedPath[]> annotatedPathSourceBlock, ITargetBlock<IFileFingerprint> fileFingerprintTargetBlock, CancellationToken cancellationToken) { try { var targets = new ConcurrentDictionary<string, TransformBlock<AnnotatedPath, IFileFingerprint>>(StringComparer.InvariantCultureIgnoreCase); var routeBlock = new ActionBlock<AnnotatedPath[]>( async filenames => { foreach (var filename in filenames) { if (null == filename) continue; var cachedBlob = GetCachedFileFingerprint(filename.FileInfo); if (null != cachedBlob) { await fileFingerprintTargetBlock.SendAsync(cachedBlob, cancellationToken).ConfigureAwait(false); continue; } var host = PathUtil.GetHost(filename.FileInfo.FullName); TransformBlock<AnnotatedPath, IFileFingerprint> target; while (!targets.TryGetValue(host, out target)) { target = new TransformBlock<AnnotatedPath, IFileFingerprint>(annotatedPath => ProcessFileAsync(annotatedPath.FileInfo, cancellationToken), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = cancellationToken }); if (!targets.TryAdd(host, target)) continue; Debug.WriteLine($"FileFingerprintManager.GenerateBlobsAsync() starting reader for host: '{host}'"); target.LinkTo(fileFingerprintTargetBlock, blob => null != blob); target.LinkTo(DataflowBlock.NullTarget<IFileFingerprint>()); break; } //Debug.WriteLine($"FileFingerprintManager.GenerateFileFingerprintsAsync() Sending {annotatedPath} for host '{host}'"); await target.SendAsync(filename, cancellationToken).ConfigureAwait(false); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 16, CancellationToken = cancellationToken }); var distinctPaths = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); var distinctBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>( annotatedPaths => { for (var i = 0; i < annotatedPaths.Length; ++i) { if (!distinctPaths.Add(annotatedPaths[i].FileInfo.FullName)) annotatedPaths[i] = null; } return annotatedPaths; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, CancellationToken = cancellationToken }); distinctBlock.LinkTo(routeBlock, new DataflowLinkOptions { PropagateCompletion = true }); annotatedPathSourceBlock.LinkTo(distinctBlock, new DataflowLinkOptions { PropagateCompletion = true }); await routeBlock.Completion.ConfigureAwait(false); foreach (var target in targets.Values) target.Complete(); await Task.WhenAll(targets.Values.Select(target => target.Completion)); } catch (Exception ex) { Console.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() failed: " + ex.Message); } finally { Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() is done"); fileFingerprintTargetBlock.Complete(); } }
// Demonstrates the consumption end of the producer and consumer pattern. public async Task<Parse> AscTransConsumerAsync(ISourceBlock<Parse> source) { // counter to track the number of items that are processed Int64 count = 0; var parse = new Parse(); var actionBlock = new ActionBlock<Parse>( data => { ProcessDataBuffer(data); // count has to be accessed in a thread-safe manner // be careful about using Interlocked, // for more complicated computations, locking might be more appropriate Interlocked.Increment(ref count); }, // some small constant might be better than Unbounded, depedning on circumstances new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 5}); //DataflowBlockOptions.Unbounded}); source.LinkTo(actionBlock, new DataflowLinkOptions {PropagateCompletion = true}); // this assumes source will be completed when done, // you need to call ascbuffer.Complete() after AscBufferProducer() for this await actionBlock.Completion; return parse; }
async Task ProcessMessages(ISourceBlock<Message> queue) //обрабатываю сообщения в буфере { Message message; while (await queue.OutputAvailableAsync()) { message = (await queue.ReceiveAsync()); User user; try { using (var repository = new Repository<DutyBotDbContext>()) { user = repository.Get<User>(u => u.TlgNumber == message.chat.id); if (user == null) { user = new User { Name = message.chat.first_name, TlgNumber = message.chat.id, State = -1, Login = "", Password = "", TicketNumber = "" }; repository.Create(user); } } } catch (Exception) { _bot.SendMessage(message.chat.id, "Что-то пошло не так при обращении к базе данных. Дальнейшая работа не возможна."); Thread.Sleep(5000); return; } try { using (var repository = new Repository<DutyBotDbContext>()) { user = repository.Get<User>(u => u.Id == user.Id); switch (user.State) //смотрю, в каком статусе пользователь //- 1 его нет, 0 ззнакомство с DUtyBot, 1 Ввод пароля, 2, проверка доступа в jira, //3 основной статус, ожидаем команды, 4 пользователь решает что делать с тикетом, который получил от бота по кнопке Проверь тикеты //5 идёт мониторинг тикетов, пользователь получает уведомления, 6 пользователь решает что делать с тикетом, который получил при мониторинге { case -1: _bot.SendMessage(message.chat.id, "Привет, " + message.chat.first_name + @"! Меня зовут DutyBot, я создан, чтобы помогать дежурить. Давай знакомиться! ", "{\"keyboard\": [[\"Рассказать DutyBot'у о себе\"], [\"Не хочу знакомиться, ты мне не нравишься\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); user.State = 0; break; case 0: if ((message.text) == "Рассказать DutyBot'у о себе" | (message.text) == "Ввести учётку еще раз") { _bot.SendMessage(message.chat.id, @"Чтобы мы могли работать в Jira, мне нужны твои учётные данные. Напиши мне свой логин в формате d.bot и нажми отправить. Твои данные будут относительно безопасно храниться на сервере"); user.Name = message.chat.first_name; user.TlgNumber = message.chat.id; user.State = 1; break; } if ((message.text) == "Не хочу знакомиться, ты мне не нравишься") { _bot.SendMessage(message.chat.id, @"Очень жаль, но если надумешь, пиши. Я забуду об этом неприятном разговоре"); repository.Delete(user); } break; case 1: _bot.SendMessage(message.chat.id, @"Теперь напиши пароль"); user.Login = message.text; user.State = 2; break; case 2: user.Password = message.text; _bot.SendMessage(message.chat.id, @"Отлично, сейчас я проверю, есть ли у тебя доступ в Jira"); if (JiraAddFuncions.CheckConnection(_jiraConn, user.Login, user.Password)) { _bot.SendMessage(message.chat.id, @"Всё хорошо, можно начинать работу", "{\"keyboard\": [[\"Начнём\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); user.Password = message.text; user.State = 3; break; } _bot.SendMessage(message.chat.id, @"Доступа к JIra нет. Возможно учётные данные не верны. Давай попробуем ввести их еще раз. ", "{\"keyboard\": [[\"Ввести учётку еще раз\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); user.State = 0; break; case 3: switch (message.text) { case "Начнём": _bot.SendMessage(message.chat.id, "Просто напиши мне сообщение, когда я тебе понадоблюсь. В ответ я пришлю меню с вариантами моих действий. Вот такое ↓", "{\"keyboard\": [[\"Кто сейчас дежурит?\"], [\"Проверь тикеты\"], [\"Помоги с дежурством\"], [\"Пока ничем\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; case "Кто сейчас дежурит?": _bot.SendMessage(message.chat.id, DbReader.Readrespersone(), "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; case "Проверь тикеты": _jiraConn = Jira.CreateRestClient(_jiraParam.Value, user.Login, user.Password); try { var issues = _jiraConn.GetIssuesFromJql(_filterParam.Value); IList<Issue> enumerable = issues as IList<Issue> ?? issues.ToList(); if (enumerable.Any()) { _ticket = enumerable.Last(); user.TicketNumber = _ticket.Key.ToString(); user.State = 4; _bot.SendMessage(message.chat.id, _ticket); } else { _bot.SendMessage(message.chat.id, "Тикетов нет", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); } } catch (Exception ex) { _bot.SendMessage(message.chat.id, "Jira не доступна", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); var logReccord = new Log { Date = DateTime.Now, MessageTipe = "error", UserId = 0, Operation = "SendThatJiraDoesNotWork", Exception = ex.GetType() + ": " + ex.Message, AddInfo = "" }; repository.Create(logReccord); } break; case "Помоги с дежурством": _bot.SendMessage(message.chat.id, "Как будем дежурить?", "{\"keyboard\": [[\"Начать мониторить тикеты\"], [\"Мониторить тикеты в моё дежурство\"], [\"Отмена\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; case "Пока ничего": _bot.SendMessage(message.chat.id, "Ок, если что, пиши."); break; case "Начать мониторить тикеты": _bot.SendMessage(message.chat.id, @"Начинаю мониторинг. Я буду мониторить тикеты в течение ближайших 12 часов, после чего мониторинг будет автоматически остановлен."); user.State = 5; user.DutyStart = DateTime.Now; user.DutyEnd = DateTime.Now.AddHours(12); break; case "Мониторить тикеты в моё дежурство": if (DbReader.Readuserdutystart(message.chat.id) == Convert.ToDateTime("01.01.1900 0:00:00 ") | DbReader.Readuserdutystart(message.chat.id) == Convert.ToDateTime("01.01.1900 0:00:00 ")) { _bot.SendMessage(message.chat.id, "У тебя нет дежурств в ближайшее время", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; } _bot.SendMessage(message.chat.id, "Я буду мониторить тикеты с " + DbReader.Readuserdutystart(message.chat.id).ToShortDateString() + " " + DbReader.Readuserdutystart(message.chat.id).ToShortTimeString() + " по " + DbReader.Readuserdutyend(message.chat.id).ToShortDateString() + " " + DbReader.Readuserdutyend(message.chat.id).ToShortTimeString()); user.State = 5; user.DutyStart = DbReader.Readuserdutystart(message.chat.id); user.DutyEnd = DbReader.Readuserdutyend(message.chat.id); break; default: _bot.SendMessage(message.chat.id, "Чем я могу помочь?", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; } break; case 4: if (_ticket.Assignee == null & _ticket.Key.ToString().Equals(user.TicketNumber)) { switch ((message.text)) { case "Распределить": _bot.SendMessage(message.chat.id, "Кому назначим?", "{\"keyboard\": [[\"Технологи\", \"Коммерция\"], [\"Админы\", \"Связисты\"], [\"Олеся\", \"Женя\"], [\"Алексей\", \"Максим\"], [\"Паша\", \"Марина\"], [\"Андрей\", \"Гриша\"], [\"Оля\", \"Настя\"], [\"Маркетинг\", \"Даша\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; case "Решить": JiraAddFuncions.ResolveTicket(user, _ticket, message, user.Login, _bot, _jiraConn); break; case "Назначить себе": JiraAddFuncions.AssingTicket(user, _ticket, message, user.Login, _bot, _jiraConn); break; case "Технологи": JiraAddFuncions.AssingTicket(user, _ticket, message, "technologsupport", _bot, _jiraConn); break; case "Коммерция": JiraAddFuncions.AssingTicket(user, _ticket, message, "crm_otdel", _bot, _jiraConn); break; case "Админы": JiraAddFuncions.AssingTicket(user, _ticket, message, "Uk.Jira.TechSupport", _bot, _jiraConn); break; case "Связисты": JiraAddFuncions.AssingTicket(user, _ticket, message, "uk.jira.noc", _bot, _jiraConn); break; case "Олеся": JiraAddFuncions.AssingTicket(user, _ticket, message, "o.likhacheva", _bot, _jiraConn); break; case "Женя": JiraAddFuncions.AssingTicket(user, _ticket, message, "ev.safonov", _bot, _jiraConn); break; case "Алексей": JiraAddFuncions.AssingTicket(user, _ticket, message, "a.sapotko", _bot, _jiraConn); break; case "Максим": JiraAddFuncions.AssingTicket(user, _ticket, message, "m.shemetov", _bot, _jiraConn); break; case "Андрей": JiraAddFuncions.AssingTicket(user, _ticket, message, "an.zarubin", _bot, _jiraConn); break; case "Гриша": JiraAddFuncions.AssingTicket(user, _ticket, message, "g.dementiev", _bot, _jiraConn); break; case "Оля": JiraAddFuncions.AssingTicket(user, _ticket, message, "o.tkachenko", _bot, _jiraConn); break; case "Настя": JiraAddFuncions.AssingTicket(user, _ticket, message, "a.zakharova", _bot, _jiraConn); break; case "Марина": JiraAddFuncions.AssingTicket(user, _ticket, message, "m.vinnikova", _bot, _jiraConn); break; case "Паша": JiraAddFuncions.AssingTicket(user, _ticket, message, "p.denisov", _bot, _jiraConn); break; case "Даша": JiraAddFuncions.AssingTicket(user, _ticket, message, "d.kormushina", _bot, _jiraConn); break; case "Маркетинг": JiraAddFuncions.AssingTicket(user, _ticket, message, "Uk.Jira.TradeMarketing", _bot, _jiraConn); break; default: _bot.SendMessage(message.chat.id, "", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); user.State = 3; break; } } else { _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён.", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); user.State = 3; } break; case 5: switch (message.text) { case ("Ок"): break; case ("Остановить мониторинг"): _bot.SendMessage(message.chat.id, "Готово"); user.State = 3; break; case ("Продолжить мониторинг"): _bot.SendMessage(message.chat.id, "Хорошо, продолжим"); break; default: _bot.SendMessage(message.chat.id, "да?", "{\"keyboard\": [[\"Остановить мониторинг\"], [\"Продолжить мониторинг\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; } break; case 6: switch (message.text) { case "Распределить": if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber)) { _bot.SendMessage(message.chat.id, "Кому назначим?", "{\"keyboard\": [[\"Технологи\", \"Коммерция\"], [\"Админы\", \"Связисты\"], [\"Олеся\", \"Женя\"], [\"Алексей\", \"Максим\"], [\"Паша\", \"Марина\"], [\"Андрей\", \"Гриша\"], [\"Оля\", \"Настя\"], [\"Маркетинг\", \"Даша\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; } _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён."); user.State = 5; break; case "Решить": if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber)) { JiraAddFuncions.ResolveTicket(user, lastIssue, message, user.Login, _bot, _jiraConn); break; } _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён."); user.State = 5; break; case "Назначить себе": if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber)) { JiraAddFuncions.AssingTicket(user, lastIssue, message, user.Login, _bot, _jiraConn); break; } _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён."); user.State = 5; break; case "Технологи": JiraAddFuncions.AssingTicket(user, lastIssue, message, "technologsupport", _bot, _jiraConn); break; case "Коммерция": JiraAddFuncions.AssingTicket(user, lastIssue, message, "crm_otdel", _bot, _jiraConn); break; case "Админы": JiraAddFuncions.AssingTicket(user, lastIssue, message, "Uk.Jira.TechSupport", _bot, _jiraConn); break; case "Связисты": JiraAddFuncions.AssingTicket(user, lastIssue, message, "uk.jira.noc", _bot, _jiraConn); break; case "Олеся": JiraAddFuncions.AssingTicket(user, lastIssue, message, "o.likhacheva", _bot, _jiraConn); break; case "Женя": JiraAddFuncions.AssingTicket(user, lastIssue, message, "ev.safonov", _bot, _jiraConn); break; case "Алексей": JiraAddFuncions.AssingTicket(user, lastIssue, message, "a.sapotko", _bot, _jiraConn); break; case "Максим": JiraAddFuncions.AssingTicket(user, lastIssue, message, "m.shemetov", _bot, _jiraConn); break; case "Андрей": JiraAddFuncions.AssingTicket(user, lastIssue, message, "an.zarubin", _bot, _jiraConn); break; case "Гриша": JiraAddFuncions.AssingTicket(user, lastIssue, message, "g.dementiev", _bot, _jiraConn); break; case "Оля": JiraAddFuncions.AssingTicket(user, lastIssue, message, "o.tkachenko", _bot, _jiraConn); break; case "Настя": JiraAddFuncions.AssingTicket(user, lastIssue, message, "a.zakharova", _bot, _jiraConn); break; case "Марина": JiraAddFuncions.AssingTicket(user, lastIssue, message, "m.vinnikova", _bot, _jiraConn); break; case "Паша": JiraAddFuncions.AssingTicket(user, lastIssue, message, "p.denisov", _bot, _jiraConn); break; case "Маркетинг": JiraAddFuncions.AssingTicket(user, lastIssue, message, "Uk.Jira.TradeMarketing", _bot, _jiraConn); break; case "Даша": JiraAddFuncions.AssingTicket(user, lastIssue, message, "d.kormushina", _bot, _jiraConn); break; case "Остановить мониторинг": { user.State = 3; _bot.SendMessage(message.chat.id, "Готово"); break; } case "Продолжить мониторинг": { user.State = 5; _bot.SendMessage(message.chat.id, "Хорошо, продолжим"); break; } default: _bot.SendMessage(message.chat.id, "да?", "{\"keyboard\": [[\"Остановить мониторинг\"], [\"Продолжить мониторинг\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); break; } break; } repository.Update(); } } catch (Exception ex) { using (var repository = new Repository<DutyBotDbContext>()) { var RepUser = repository.Get<User>(usr => usr.Id == user.Id); if (RepUser.State > 3) RepUser.State = 3; _bot.SendMessage(message.chat.id, "Что-то пошло не так при обработке сообщения. Что будем делать?", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}"); repository.Update(); var logReccord = new Log { Date = DateTime.Now, MessageTipe = "info", UserId = message.chat.id, Operation = "ProcessMessage", Exception = ex.Message, AddInfo = "" }; repository.Create(logReccord); } Thread.Sleep(5000); } } }
public ISourceBlock<ValidatableTx> CalculateUtxo(IChainStateCursor chainStateCursor, Chain chain, ISourceBlock<DecodedBlockTx> blockTxes, CancellationToken cancelToken = default(CancellationToken)) { var chainedHeader = chain.LastBlock; var blockSpentTxes = new BlockSpentTxesBuilder(); var utxoCalculator = new TransformBlock<DecodedBlockTx, ValidatableTx>( blockTx => { var tx = blockTx.Transaction; var txIndex = blockTx.Index; var prevTxOutputs = ImmutableArray.CreateBuilder<PrevTxOutput>(!blockTx.IsCoinbase ? tx.Inputs.Length : 0); //TODO apply real coinbase rule // https://github.com/bitcoin/bitcoin/blob/481d89979457d69da07edd99fba451fd42a47f5c/src/core.h#L219 if (!blockTx.IsCoinbase) { // spend each of the transaction's inputs in the utxo for (var inputIndex = 0; inputIndex < tx.Inputs.Length; inputIndex++) { var input = tx.Inputs[inputIndex]; var prevTxOutput = this.Spend(chainStateCursor, txIndex, tx, inputIndex, input, chainedHeader, blockSpentTxes); prevTxOutputs.Add(prevTxOutput); } } // there exist two duplicate coinbases in the blockchain, which the design assumes to be impossible // ignore the first occurrences of these duplicates so that they do not need to later be deleted from the utxo, an unsupported operation // no other duplicates will occur again, it is now disallowed var isDupeCoinbase = IsDupeCoinbase(chainedHeader, tx); // add transaction's outputs to utxo, except for the genesis block and the duplicate coinbases if (chainedHeader.Height > 0 && !isDupeCoinbase) { // mint the transaction's outputs in the utxo this.Mint(chainStateCursor, tx, txIndex, chainedHeader); // increase unspent output count chainStateCursor.UnspentOutputCount += tx.Outputs.Length; // increment unspent tx count chainStateCursor.UnspentTxCount++; chainStateCursor.TotalTxCount++; chainStateCursor.TotalInputCount += tx.Inputs.Length; chainStateCursor.TotalOutputCount += tx.Outputs.Length; } return new ValidatableTx(blockTx, chainedHeader, prevTxOutputs.MoveToImmutable()); }, new ExecutionDataflowBlockOptions { CancellationToken = cancelToken }); blockTxes.LinkTo(utxoCalculator, new DataflowLinkOptions { PropagateCompletion = true }); return OnCompleteBlock.Create(utxoCalculator, () => { if (!chainStateCursor.TryAddBlockSpentTxes(chainedHeader.Height, blockSpentTxes.ToImmutable())) throw new ValidationException(chainedHeader.Hash); }, cancelToken); }
private async System.Threading.Tasks.Task WriteOutputAsync(ISourceBlock<string> source) { while (await source.OutputAvailableAsync()) { string data = source.Receive(); if (OutputPane != null) { OutputPane.OutputStringThreadSafe(data); OutputPane.OutputStringThreadSafe("\r\n"); } } }
public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket messageValue, ISourceBlock<Bucket> source, bool consumeToAccept) { var lines = messageValue.ToLines(); _bufferBlock.Post(lines); return DataflowMessageStatus.Accepted; }
public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket messageValue, ISourceBlock<Bucket> source, bool consumeToAccept) { _preprocessorBlock.Post(messageValue); return DataflowMessageStatus.Accepted; }
public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket messageValue, ISourceBlock<Bucket> source, bool consumeToAccept) { messageValue.FeedTarget(_batchBlock); return DataflowMessageStatus.Accepted; }
public Task UploadBlobsAsync(AwsManager awsManager, ISourceBlock<Tuple<IFileFingerprint, AnnotatedPath>> uniqueBlobBlock, IReadOnlyDictionary<string, string> knowObjects, CancellationToken cancellationToken) { var blobCount = 0; var blobTotalSize = 0L; var builderBlock = new TransformBlock<Tuple<IFileFingerprint, AnnotatedPath>, S3Blobs.IUploadBlobRequest>( tuple => { string etag; var exists = knowObjects.TryGetValue(tuple.Item1.Fingerprint.Key(), out etag); //Debug.WriteLine($"{tuple.Item1.FullFilePath} {(exists ? "already exists" : "scheduled for upload")}"); if (exists) { // We can't check multipart uploads this way since we don't know the size // of the individual parts. if (etag.Contains("-")) { Debug.WriteLine($"{tuple.Item1.FullFilePath} is a multi-part upload with ETag {etag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}"); return null; } var expectedETag = tuple.Item1.Fingerprint.S3ETag(); if (string.Equals(expectedETag, etag, StringComparison.InvariantCultureIgnoreCase)) return null; Console.WriteLine($"ERROR: {tuple.Item1.FullFilePath} tag mismatch {etag}, expected {expectedETag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}"); } var request = awsManager.BuildUploadBlobRequest(tuple); if (null == request) return null; Interlocked.Increment(ref blobCount); Interlocked.Add(ref blobTotalSize, request.FileFingerprint.Fingerprint.Size); return request; }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount }); var uploader = new ActionBlock<S3Blobs.IUploadBlobRequest>( blob => UploadBlobAsync(awsManager, blob, cancellationToken), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken }); builderBlock.LinkTo(uploader, new DataflowLinkOptions { PropagateCompletion = true }, r => null != r); builderBlock.LinkTo(DataflowBlock.NullTarget<S3Blobs.IUploadBlobRequest>()); uniqueBlobBlock.LinkTo(builderBlock, new DataflowLinkOptions { PropagateCompletion = true }); var tasks = new List<Task>(); #if DEBUG var uploadDoneTask = uploader.Completion .ContinueWith( _ => Debug.WriteLine($"Done uploading blobs: {blobCount} items {SizeConversion.BytesToGiB(blobTotalSize):F2}GiB"), cancellationToken); tasks.Add(uploadDoneTask); #endif tasks.Add(uploader.Completion); return Task.WhenAll(tasks); }
/// <summary> /// Initializes a new instance of the <see cref="ArrayDataReader" /> class. /// </summary> /// <param name="sourceBlock">The source block.</param> /// <param name="fieldCount">The field count.</param> public ArrayDataReader(ISourceBlock<object> sourceBlock, int fieldCount) { _sourceBlock = sourceBlock; FieldCount = fieldCount; }
public AsyncWordVectorLoader(string filename) : base(filename) { _linesSource = ReadWordVecLines(filename); }
/// <summary> /// Offers the message. /// </summary> /// <param name="messageHeader">The message header.</param> /// <param name="messageValue">The message value.</param> /// <param name="source">The source.</param> /// <param name="consumeToAccept">The consume to accept.</param> /// <returns>DataflowMessageStatus.</returns> public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, LogEntry messageValue, ISourceBlock<LogEntry> source, Boolean consumeToAccept) { return _BatchBlock.OfferMessage(messageHeader, messageValue, source, consumeToAccept); }