Esempio n. 1
0
        public void Consumer(ISourceBlock<string> source)
        {
            var ablock = new ActionBlock<string>(
                data => this.ParseRawIRC(data), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });

            source.LinkTo(ablock);
        }
Esempio n. 2
0
        public static async Task ValidateBlockAsync(ICoreStorage coreStorage, ICoreRules rules, Chain newChain, ISourceBlock<ValidatableTx> validatableTxes, CancellationToken cancelToken = default(CancellationToken))
        {
            // tally transactions
            object finalTally = null;
            var txTallier = new TransformBlock<ValidatableTx, ValidatableTx>(
                validatableTx =>
                {
                    var runningTally = finalTally;
                    rules.TallyTransaction(newChain, validatableTx, ref runningTally);
                    finalTally = runningTally;

                    return validatableTx;
                });
            validatableTxes.LinkTo(txTallier, new DataflowLinkOptions { PropagateCompletion = true });

            // validate transactions
            var txValidator = InitTxValidator(rules, newChain, cancelToken);

            // begin feeding the tx validator
            txTallier.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true });

            // validate scripts
            var scriptValidator = InitScriptValidator(rules, newChain, cancelToken);

            // begin feeding the script validator
            txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true });

            //TODO
            await PipelineCompletion.Create(
                new Task[] { },
                new IDataflowBlock[] { validatableTxes, txTallier, txValidator, scriptValidator });

            // validate overall block
            rules.PostValidateBlock(newChain, finalTally);
        }
        public void CalculateAverage(ISourceBlock<List<Citizen>> source, Action<int, int> PrintAverage)
        {
            if(extractionCompletion == null)
            {
                ExtractData(source);
            }

            extractionCompletion.ContinueWith((tsk) => PrintAverage(totalAge, peopleCount));
        }
Esempio n. 4
0
        public async Task UpdateLinksAsync(AwsManager awsManager, ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> linkBlobs,
            CancellationToken cancellationToken)
        {
            try
            {
                await _linkManager.CreateLinksAsync(awsManager, linkBlobs, _s3Settings.ActuallyWrite, cancellationToken).ConfigureAwait(false);

                Debug.WriteLine("Done processing links");
            }
            catch (Exception ex)
            {
                Debug.WriteLine("Processing links failed: " + ex.Message);
            }
        }
Esempio n. 5
0
 /// <summary>
 /// 初期化処理。
 /// </summary>
 /// <param name="settingObject"></param>
 /// <param name="token"></param>
 public void Init(dynamic settingObject, CancellationToken token)
 {
     logger.Trace("Init Start");
     var opt = new DataflowBlockOptions
     {
         CancellationToken = token,
     };
     var buffer = new BufferBlock<PastaLog>(opt);
     var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt);
     buffer.LinkTo(bloadcast);
     Target = buffer;
     Source = bloadcast;
     logger.Trace("Init End");
 }
Esempio n. 6
0
        async Task CreateLinksBlockAsync(AwsManager awsManager, string collection,
            ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> collectionBlock, bool actuallyWrite, CancellationToken cancellationToken)
        {
            var links = await awsManager.GetLinksAsync(collection, cancellationToken).ConfigureAwait(false);

            Debug.WriteLine($"Link handler for {collection} found {links.Count} existing links");

            var createLinkBlock = new ActionBlock<S3Links.ICreateLinkRequest>(
                link => CreateLinkAsync(awsManager, link, actuallyWrite, cancellationToken),
                new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 512, CancellationToken = cancellationToken });

            var makeLinkBlock = new TransformBlock<Tuple<AnnotatedPath, IFileFingerprint>, S3Links.ICreateLinkRequest>(
                tuple =>
                {
                    var path = tuple.Item1;
                    var file = tuple.Item2;

                    if (collection != path.Collection)
                        throw new InvalidOperationException($"Create link for {path.Collection} on {collection}");

                    var relativePath = path.RelativePath;

                    if (relativePath.StartsWith(".."))
                        throw new InvalidOperationException($"Create link for invalid path {relativePath}");

                    if (relativePath.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
                        throw new InvalidOperationException($"Create link for invalid path {relativePath}");

                    relativePath = relativePath.Replace('\\', '/');

                    if (relativePath.StartsWith("/", StringComparison.Ordinal))
                        throw new InvalidOperationException($"Create link for invalid path {relativePath}");

                    string eTag;
                    links.TryGetValue(relativePath, out eTag);

                    return awsManager.BuildLinkRequest(collection, relativePath, file, eTag);
                },
                new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });

            makeLinkBlock.LinkTo(createLinkBlock, new DataflowLinkOptions { PropagateCompletion = true }, link => null != link);
            makeLinkBlock.LinkTo(DataflowBlock.NullTarget<S3Links.ICreateLinkRequest>());

            collectionBlock.LinkTo(makeLinkBlock, new DataflowLinkOptions { PropagateCompletion = true });

            await createLinkBlock.Completion.ConfigureAwait(false);

            Debug.WriteLine($"Link handler for {collection} is done");
        }
 public void FiveMostUncommonNames(ISourceBlock<List<Citizen>> source, Action<List<string>, string> PrintResult)
 {
     if (extractionCompletion == null)
     {
         ExtractData(source);
     }
     extractionCompletion.ContinueWith(
     (arg) =>
     {
         var sortedDict = from entry in globalNameFrequency orderby entry.Value ascending select entry.Key;
         var list = sortedDict.Take(5).ToList();
         PrintResult(list, "uncommon");
     }
     );
 }
Esempio n. 8
0
        public static ISourceBlock<LoadedTx> LoadTxes(ICoreStorage coreStorage, ISourceBlock<LoadingTx> loadingTxes, CancellationToken cancelToken = default(CancellationToken))
        {
            // split incoming LoadingTx by its number of inputs
            var createTxInputList = InitCreateTxInputList(cancelToken);

            // link the loading txes to the input splitter
            loadingTxes.LinkTo(createTxInputList, new DataflowLinkOptions { PropagateCompletion = true });

            // load each input, and return and fully loaded txes
            var loadTxInputAndReturnLoadedTx = InitLoadTxInputAndReturnLoadedTx(coreStorage, cancelToken);

            // link the input splitter to the input loader
            createTxInputList.LinkTo(loadTxInputAndReturnLoadedTx, new DataflowLinkOptions { PropagateCompletion = true });

            return loadTxInputAndReturnLoadedTx;
        }
Esempio n. 9
0
        public static ISourceBlock<BlockTx> LookAhead(ISourceBlock<BlockTx> blockTxes, IDeferredChainStateCursor deferredChainStateCursor, CancellationToken cancelToken = default(CancellationToken))
        {
            // capture the original block txes order
            var orderedBlockTxes = OrderingBlock.CaptureOrder<BlockTx, BlockTx, int>(
                blockTxes, blockTx => blockTx.Index, cancelToken);

            // queue each utxo entry to be warmed up: each input's previous transaction, and each new transaction
            var queueUnspentTxLookup = InitQueueUnspentTxLookup(cancelToken);
            orderedBlockTxes.LinkTo(queueUnspentTxLookup, new DataflowLinkOptions { PropagateCompletion = true });

            // warm up each uxto entry
            var warmupUtxo = InitWarmupUtxo(deferredChainStateCursor, cancelToken);
            queueUnspentTxLookup.LinkTo(warmupUtxo, new DataflowLinkOptions { PropagateCompletion = true });

            // return the block txes with warmed utxo entries in original order
            return orderedBlockTxes.ApplyOrder(warmupUtxo, blockTx => blockTx.Index, cancelToken);
        }
Esempio n. 10
0
        public async Task CreateLinksAsync(AwsManager awsManager, ISourceBlock<Tuple<AnnotatedPath, IFileFingerprint>> blobSourceBlock,
            bool actuallyWrite, CancellationToken cancellationToken)
        {
            var collectionBlocks = new Dictionary<string, ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>>>();
            var tasks = new List<Task>();

            var routeBlock = new ActionBlock<Tuple<AnnotatedPath, IFileFingerprint>>(async blob =>
            {
                var collection = blob.Item1.Collection;

                if (string.IsNullOrEmpty(collection))
                    return;

                ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> collectionBlock;
                if (!collectionBlocks.TryGetValue(collection, out collectionBlock))
                {
                    var bufferBlock = new BufferBlock<Tuple<AnnotatedPath, IFileFingerprint>>();

                    collectionBlock = bufferBlock;

                    collectionBlocks[collection] = collectionBlock;

                    var task = CreateLinksBlockAsync(awsManager, collection, bufferBlock, actuallyWrite, cancellationToken);

                    tasks.Add(task);
                }

                await collectionBlock.SendAsync(blob, cancellationToken).ConfigureAwait(false);
            });

            blobSourceBlock.LinkTo(routeBlock, new DataflowLinkOptions { PropagateCompletion = true });

            await routeBlock.Completion.ConfigureAwait(false);

            Debug.WriteLine("S3LinkCreateor.CreateLinkAsync() routeBlock is done");

            foreach (var block in collectionBlocks.Values)
                block.Complete();

            await Task.WhenAll(tasks).ConfigureAwait(false);

            Debug.WriteLine("S3LinkCreateor.CreateLinkAsync() all link blocks are done");
        }
Esempio n. 11
0
        public async Task WorkAsync(ISourceBlock <TEntity> source, CancellationToken c)
        {
            var bufferSize = _options
                             .Value.BufferSize.GetOrDefault(typeof(TEntity).Name, _options.Value.DefaultBufferSize);
            var buffer = new TEntity[bufferSize];

            _logger.LogInformation("Start working, buffer size: {0}", bufferSize);

            while (await source.OutputAvailableAsync(c))
            {
                var i = 0;
                for (;
                     i < bufferSize && await source.OutputAvailableAsync(c);
                     i++)
                {
                    buffer[i] = await source.ReceiveAsync(c);
                }

                await Insert(buffer.Take(i));
            }
        }
        /// <summary>
        /// Pipes the source data through a single producer constrained ActionBlock.
        /// </summary>
        /// <typeparam name="T">The input type.</typeparam>
        /// <param name="source">The source block to receive from.</param>
        /// <param name="handler">The handler function to apply.</param>
        /// <param name="options">Optional execution options.</param>
        /// <returns>The ActionBlock created.</returns>
        public static ActionBlock <T> Pipe <T>(this ISourceBlock <T> source,
                                               Action <T> handler,
                                               ExecutionDataflowBlockOptions?options = null)
        {
            if (source is null)
            {
                throw new NullReferenceException();
            }
            if (handler is null)
            {
                throw new ArgumentNullException(nameof(handler));
            }
            Contract.EndContractBlock();

            var receiver = options is null
                                ? new ActionBlock <T>(handler)
                                : new ActionBlock <T>(handler, options);

            source.LinkToWithCompletion(receiver);
            return(receiver);
        }
Esempio n. 13
0
        public ServiceDiscovery(string serviceName,
                                ReachabilityChecker reachabilityChecker,
                                IRemoteHostPoolFactory remoteHostPoolFactory,
                                IDiscoverySourceLoader serviceDiscoveryLoader,
                                IEnvironment environment,
                                ISourceBlock <DiscoveryConfig> configListener,
                                Func <DiscoveryConfig> discoveryConfigFactory,
                                ILog log)
        {
            Log                    = log;
            _serviceName           = serviceName;
            _originatingDeployment = new DeploymentIdentifier(serviceName, environment.DeploymentEnvironment, environment);
            _masterDeployment      = new DeploymentIdentifier(serviceName, MASTER_ENVIRONMENT, environment);

            _reachabilityChecker    = reachabilityChecker;
            _remoteHostPoolFactory  = remoteHostPoolFactory;
            _serviceDiscoveryLoader = serviceDiscoveryLoader;

            _initTask        = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory()));
            _configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost));
        }
Esempio n. 14
0
        public BasePingParser(Func <string, Ping> parse)
        {
            _source = new BufferBlock <Ping>();
            _target = new ActionBlock <string>(async msg => await((BufferBlock <Ping>)_source).SendAsync(parse(msg)),
                                               new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
            });

            _target.Completion.ContinueWith((t) =>
            {
                if (t.IsFaulted)
                {
                    _source.Fault(t.Exception);
                }
                else
                {
                    _source.Complete();
                }
            });
        }
        public static async void ConsumeMediaAsync(string path, ISourceBlock <KeyValuePair <string, string> > source)
        {
            if (!File.Exists(path))
            {
                Directory.CreateDirectory(path);
            }
            var filesProcessed = 0; var filesDownloaded = 0;

            while (await source.OutputAvailableAsync())
            {
                var client = new WebClient();
                var(fileName, fileUri) = source.Receive();
                filesProcessed++;

                //Conflicted about whether or not this should exist. Doesn't save time, but can fix (or break)
                //previous downloads.
                Logger.Info("ConsumeMediaAsync|" + filesProcessed + " Processing: " + fileName);
                var fileExists = Directory.GetFiles(path, fileName + ".*");
                if (fileExists.Length > 0)
                {
                    continue;
                }

                Logger.Info("ConsumeMediaAsync|" + filesProcessed + " Downloading: " + fileName);

                if (fileUri.Contains(".mp4"))
                {
                    client.DownloadFileAsync(new Uri(fileUri), path + fileName + ".mp4");
                }
                else
                {
                    client.DownloadFileAsync(new Uri(fileUri), path + fileName + ".jpg");
                }

                filesDownloaded++;
            }

            Logger.Info("ConsumeMediaAsync|" + "Processed {0} files.", filesProcessed);
            Logger.Info("ConsumeMediaAsync|" + "Downloaded {0} files.", filesDownloaded);
        }
 public async Task <int> SaveFileAsync(ISourceBlock <string> source)
 {
     while (await source.OutputAvailableAsync())
     {
         WebClient wc = new WebClient();
         if (!Directory.Exists("img"))
         {
             Directory.CreateDirectory("img");
         }
         var imageUrl = source.Receive();
         if (!imageUrl.StartsWith(GetIhChina.MainPage))
         {
             imageUrl = GetIhChina.MainPage + imageUrl;
         }
         var savePath = Path.Combine(Directory.GetCurrentDirectory(), "img", WebpageHelper.GetSubUrl(imageUrl));
         if (File.Exists(savePath))
         {
             continue;
         }
         try
         {
             Console.WriteLine("Starting to save image {0}", imageUrl);
             wc.DownloadFile(imageUrl, savePath);
             //CompressImageWithGuetzli(savePath); 压缩貌似有点问题,待修复
             CorpImage(savePath);
             Console.WriteLine("Save image {0} completely", imageUrl);
         }
         catch (WebException e)
         {
             Console.WriteLine("Save image {0} error", imageUrl);
             Console.WriteLine(e);
         }
         catch (Exception e)
         {
             Console.WriteLine("unhandled error");
             Console.WriteLine(e);
         }
     }
     return(1);
 }
Esempio n. 17
0
        public static async Task ValidateBlockAsync(ICoreStorage coreStorage, IBlockchainRules rules, ChainedHeader chainedHeader, ISourceBlock<LoadedTx> loadedTxes, CancellationToken cancelToken = default(CancellationToken))
        {
            // validate merkle root
            var merkleStream = new MerkleStream();
            var merkleValidator = InitMerkleValidator(chainedHeader, merkleStream, cancelToken);

            // begin feeding the merkle validator
            loadedTxes.LinkTo(merkleValidator, new DataflowLinkOptions { PropagateCompletion = true });

            // validate transactions
            var txValidator = InitTxValidator(rules, chainedHeader, cancelToken);

            // begin feeding the tx validator
            merkleValidator.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true });

            // validate scripts
            var scriptValidator = InitScriptValidator(rules, chainedHeader, cancelToken);

            // begin feeding the script validator
            txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true });

            await merkleValidator.Completion;
            await txValidator.Completion;
            await scriptValidator.Completion;

            if (!rules.BypassPrevTxLoading)
            {
                try
                {
                    merkleStream.FinishPairing();
                }
                //TODO
                catch (InvalidOperationException)
                {
                    throw CreateMerkleRootException(chainedHeader);
                }
                if (merkleStream.RootNode.Hash != chainedHeader.MerkleRoot)
                    throw CreateMerkleRootException(chainedHeader);
            }
        }
Esempio n. 18
0
        public ConsumerWorkflow <TState> BuildLinkages(DataflowLinkOptions overrideOptions = null)
        {
            Guard.AgainstNull(_deserializeBlock, nameof(_deserializeBlock));
            Guard.AgainstNull(_errorBuffer, nameof(_errorBuffer));
            Guard.AgainstNull(_finalization, nameof(_finalization));

            if (_inputBuffer == null)
            {
                _inputBuffer = new BufferBlock <ReceivedData>();
            }

            if (_readyBuffer == null)
            {
                _readyBuffer = new BufferBlock <TState>();
            }

            if (_postProcessingBuffer == null)
            {
                _postProcessingBuffer = new BufferBlock <TState>();
            }

            for (int i = 0; i < ConsumerCount; i++)
            {
                _consumerBlocks.Add(new ConsumerBlock <ReceivedData>(_consumer));
                _consumerBlocks[i].LinkTo(_inputBuffer, overrideOptions ?? _linkStepOptions);
            }

            _inputBuffer.LinkTo(_deserializeBlock, overrideOptions ?? _linkStepOptions);
            _deserializeBlock.LinkTo(_errorBuffer, overrideOptions ?? _linkStepOptions, x => x == null);
            SetCurrentSourceBlock(_deserializeBlock);

            LinkPreProcessing(overrideOptions);
            LinkSuppliedSteps(overrideOptions);
            LinkPostProcessing(overrideOptions);

            _errorBuffer.LinkTo(_errorAction, overrideOptions ?? _linkStepOptions);
            _currentBlock = null;

            return(this);
        }
Esempio n. 19
0
        public EnumerableSourceCore(ISourceBlock <T> parent, Action enumerate, TaskScheduler?taskScheduler, CancellationToken cancellationToken)
        {
            this._parent            = parent;
            this._cancellationToken = cancellationToken;

            if (taskScheduler == null || taskScheduler == TaskScheduler.Default)
            {
#if THREADPOOL
                WaitCallback enumerateCb = x => ((Action)x).Invoke();
                this._enumerate = () => ThreadPool.QueueUserWorkItem(enumerateCb, enumerate);

                WaitCallback offerCb = x => ((EnumerableSourceCore <T>)x).OfferToTargets();
                this._offerToTargetsOnTaskScheduler = () => ThreadPool.QueueUserWorkItem(offerCb, this);
#else
                this._enumerate = () => Task.Run(enumerate);

                Action offerAction = this.OfferToTargets;
                this._offerToTargetsOnTaskScheduler = () => Task.Run(offerAction);
#endif
            }
            else
            {
                var taskFactory = new TaskFactory(taskScheduler);
                this._enumerate = () => taskFactory.StartNew(enumerate);

                Action offerAction = this.OfferToTargets;
                this._offerToTargetsOnTaskScheduler = () => taskFactory.StartNew(offerAction);
            }

            if (cancellationToken.IsCancellationRequested)
            {
                this.State = StateEnum.Completed;
                this._completeRequested = true;
                this._tcs.TrySetCanceled(cancellationToken);
            }
            else if (cancellationToken.CanBeCanceled)
            {
                this._cancelReg = cancellationToken.Register(() => this.Complete(false));
            }
        }
Esempio n. 20
0
            public ISourceBlock <TCombine> ApplyOrder(ISourceBlock <TCombine> source, Func <TCombine, TKey> keyFunc, CancellationToken cancelToken = default(CancellationToken))
            {
                if (sorted)
                {
                    throw new InvalidOperationException($"{GetType().Name} has already been sorted.");
                }

                // sort items back into original order
                var pendingItems = new Dictionary <TKey, TCombine>();
                var sorter       = new TransformManyBlock <TCombine, TCombine>(
                    item =>
                {
                    pendingItems.Add(keyFunc(item), item);

                    var sortedItems = new List <TCombine>();

                    // look to see if the next item in original order has been loaded
                    // if so, dequeue and return the original item and then continue looking for the next in order
                    TKey nextKey; TCombine nextItem;
                    while (orderedKeys.TryPeek(out nextKey) &&
                           pendingItems.TryGetValue(nextKey, out nextItem))
                    {
                        sortedItems.Add(nextItem);
                        pendingItems.Remove(nextKey);
                        orderedKeys.TryDequeue(out nextKey);
                    }

                    return(sortedItems);
                },
                    new ExecutionDataflowBlockOptions {
                    CancellationToken = cancelToken
                });

                source.LinkTo(sorter, new DataflowLinkOptions {
                    PropagateCompletion = true
                });

                sorted = true;
                return(sorter);
            }
Esempio n. 21
0
        private async Task BlockKeepsPostponedMessages(Func <ISourceBlock <int> > BlockFactory)
        {
            ISourceBlock <int> block  = BlockFactory();
            ITargetBlock <int> blockT = (ITargetBlock <int>)block;

            TestTargetBlock <int> testTarget = new TestTargetBlock <int>();

            testTarget.ConsumptionMode = DataflowMessageStatus.Postponed;
            block.LinkTo(testTarget, PropagateCompletion);

            // Assumption: if the target of the block is postponing messages,
            // The block will accept incoming messages until it runs out of capacity.
            Assert.True(blockT.Post(1));
            Assert.True(blockT.Post(2));
            Assert.True(blockT.Post(3));

            // Out of capacity
            Assert.False(blockT.Post(4));

            // However SendAsync() will allow postponing the message, sot the message will be eventually delivered
            blockT.SendAsync(5).Forget();

            // Wait till the block offers a message
            // Assumption: only one message will be offered, the block will not offer more messages if the target postpones
            bool messageOffered = await TaskUtils.PollWaitAsync(() => testTarget.MessagesPostponed.Count == 1, MessageArrivalTimeout);

            Assert.True(messageOffered);

            // Assumption: once the block target stops postponing, the block will keep pushing data to target
            // until it runs out of buffered messages.
            testTarget.ConsumptionMode = DataflowMessageStatus.Accepted;
            testTarget.ConsumePostponedMessages();
            // Use 10 times the normal message arrival timeout for extra padding--the test tended to be a bit flakey at this point.
            // If this happens again, switch to custom TaskScheduler.
            bool gotAllMessages = await TaskUtils.PollWaitAsync(() => testTarget.MessagesConsumed.Count == 4, TimeSpan.FromMilliseconds(MessageArrivalTimeout.TotalMilliseconds * 10));

            Assert.True(gotAllMessages, "We should have gotten 4 messages");
            Assert.Equal(testTarget.MessagesConsumed.OrderBy((i) => i), new int[] { 1, 2, 3, 5 });
        }
Esempio n. 22
0
        /// <summary>
        /// Receives an envelope from the buffer.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="buffer">The buffer.</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <returns></returns>
        /// <exception cref="System.InvalidOperationException"></exception>
        private async Task <T> ReceiveFromBufferAsync <T>(ISourceBlock <T> buffer, CancellationToken cancellationToken) where T : Envelope, new()
        {
            if (State < SessionState.Established)
            {
                throw new InvalidOperationException($"Cannot receive envelopes in the '{State}' session state");
            }

            try
            {
                return(await buffer.ReceiveAsync(cancellationToken).ConfigureAwait(false));
            }
            catch (InvalidOperationException ex) when(buffer.Completion.IsCompleted)
            {
                if (_consumeTransportTask != null)
                {
                    await _consumeTransportTask.ConfigureAwait(false);

                    _consumeTransportTask = null;
                }
                throw new InvalidOperationException("The channel listener task is complete and cannot receive envelopes", ex);
            }
        }
Esempio n. 23
0
        public DataflowMessageStatus OfferMessage(
            DataflowMessageHeader messageHeader, TOutput messageValue,
            ISourceBlock <TOutput> source, bool consumeToAccept)
        {
            if (!messageHeader.IsValid)
            {
                return(DataflowMessageStatus.Declined);
            }

            if (completion.Task.Status != TaskStatus.WaitingForActivation)
            {
                return(DataflowMessageStatus.DecliningPermanently);
            }

            lock (completion) {
                if (completion.Task.Status != TaskStatus.WaitingForActivation)
                {
                    return(DataflowMessageStatus.DecliningPermanently);
                }

                if (consumeToAccept)
                {
                    bool consummed;
                    if (!source.ReserveMessage(messageHeader, this))
                    {
                        return(DataflowMessageStatus.NotAvailable);
                    }
                    messageValue = source.ConsumeMessage(messageHeader, this, out consummed);
                    if (!consummed)
                    {
                        return(DataflowMessageStatus.NotAvailable);
                    }
                }

                completion.TrySetResult(messageValue);
            }
            CompletionSet();
            return(DataflowMessageStatus.Accepted);
        }
Esempio n. 24
0
            public DataflowMessageStatus OfferMessage(
                DataflowMessageHeader messageHeader, TSource messageValue,
                ISourceBlock <TSource> source, bool consumeToAccept)
            {
                if (consumeToAccept)
                {
                    if (!source.ReserveMessage(messageHeader, this))
                    {
                        return(DataflowMessageStatus.NotAvailable);
                    }
                    bool consumed;
                    messageValue = source.ConsumeMessage(messageHeader, this, out consumed);
                    if (!consumed)
                    {
                        return(DataflowMessageStatus.NotAvailable);
                    }
                }

                observer.OnNext(messageValue);

                return(DataflowMessageStatus.Accepted);
            }
Esempio n. 25
0
        public static Task <int> Choose <T1, T2> (ISourceBlock <T1> source1,
                                                  Action <T1> action1,
                                                  ISourceBlock <T2> source2,
                                                  Action <T2> action2,
                                                  DataflowBlockOptions dataflowBlockOptions)
        {
            if (source1 == null)
            {
                throw new ArgumentNullException("source1");
            }
            if (source2 == null)
            {
                throw new ArgumentNullException("source2");
            }

            var chooser = new ChooserBlock <T1, T2, object> (action1, action2, null, dataflowBlockOptions);

            source1.LinkTo(chooser.Target1);
            source2.LinkTo(chooser.Target2);

            return(chooser.Completion);
        }
        public ServiceDiscovery(string serviceName,
                                ReachabilityChecker reachabilityChecker,
                                IRemoteHostPoolFactory remoteHostPoolFactory,
                                IDiscoverySourceLoader serviceDiscoveryLoader,
                                IEnvironmentVariableProvider environmentVariableProvider,
                                ISourceBlock <DiscoveryConfig> configListener,
                                Func <DiscoveryConfig> discoveryConfigFactory,
                                ILog log)
        {
            Log                    = log;
            _serviceName           = serviceName;
            _originatingDeployment = new ServiceDeployment(serviceName, environmentVariableProvider.DeploymentEnvironment);
            _masterDeployment      = new ServiceDeployment(serviceName, MASTER_ENVIRONMENT);

            _reachabilityChecker    = reachabilityChecker;
            _remoteHostPoolFactory  = remoteHostPoolFactory;
            _serviceDiscoveryLoader = serviceDiscoveryLoader;

            // Must be run in Task.Run() because of incorrect Orleans scheduling
            _initTask        = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory()));
            _configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost));
        }
Esempio n. 27
0
            public DataflowMessageStatus OfferMessage(
                DataflowMessageHeader messageHeader, TMessage messageValue,
                ISourceBlock <TMessage> source, bool consumeToAccept)
            {
                if (!chooserBlock.canAccept)
                {
                    return(DataflowMessageStatus.DecliningPermanently);
                }

                bool lockTaken = false;

                try {
                    chooserBlock.messageLock.Enter(ref lockTaken);
                    if (!chooserBlock.canAccept)
                    {
                        return(DataflowMessageStatus.DecliningPermanently);
                    }

                    if (consumeToAccept)
                    {
                        bool consummed;
                        messageValue = source.ConsumeMessage(messageHeader, this, out consummed);
                        if (!consummed)
                        {
                            return(DataflowMessageStatus.NotAvailable);
                        }
                    }

                    chooserBlock.canAccept = false;
                } finally {
                    if (lockTaken)
                    {
                        chooserBlock.messageLock.Exit();
                    }
                }

                chooserBlock.MessageArrived(index, action, messageValue);
                return(DataflowMessageStatus.Accepted);
            }
Esempio n. 28
0
        public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader,
                                                  TOutput messageValue,
                                                  ISourceBlock <TOutput> source,
                                                  bool consumeToAccept)
        {
            if (!messageHeader.IsValid)
            {
                return(DataflowMessageStatus.Declined);
            }

            if (consumeToAccept)
            {
                bool consummed;
                if (!source.ReserveMessage(messageHeader, this))
                {
                    return(DataflowMessageStatus.NotAvailable);
                }
                messageValue = source.ConsumeMessage(messageHeader, this, out consummed);
                if (!consummed)
                {
                    return(DataflowMessageStatus.NotAvailable);
                }
            }

            ReceivedValue = messageValue;
            completion.TrySetResult(messageValue);
            Thread.MemoryBarrier();
            waitHandle.Set();

            /* We do the unlinking here so that we don't get called twice
             */
            if (linkBridge != null)
            {
                linkBridge.Dispose();
                linkBridge = null;
            }

            return(DataflowMessageStatus.Accepted);
        }
        public static bool TryReceiveItem <T>(this ISourceBlock <T> block, out T value)
        {
            if (block is IReceivableSourceBlock <T> receiveSourceBlock)
            {
                return(receiveSourceBlock.TryReceive(out value));
            }

            try
            {
                value = block.Receive(TimeSpan.Zero);
                return(true);
            }catch (TimeoutException)
            {
                value = default;
                return(false);
            }
            catch (InvalidOperationException)
            {
                value = default;
                return(false);
            }
        }
Esempio n. 30
0
        public DataflowMessageStatus OfferMessage(
            DataflowMessageHeader messageHeader, T messageValue, ISourceBlock <T> source, bool consumeToAccept)
        {
            if (!IsOpen)
            {
                return(DataflowMessageStatus.Declined);
            }

            if (consumeToAccept)
            {
                source.ConsumeMessage(messageHeader, this, out var messageConsumed);
                if (!messageConsumed)
                {
                    return(DataflowMessageStatus.Declined);
                }
            }

            _toPropergate = messageValue;
            _links.ForEach(target => target.OfferMessage(messageHeader, messageValue, this, true));

            return(DataflowMessageStatus.Accepted);
        }
Esempio n. 31
0
        public static OrderedSource <TInput, TOutput, TKey> CaptureOrder <TInput, TOutput, TKey>(
            ISourceBlock <TInput> source, Func <TInput, TKey> keyFunc,
            CancellationToken cancelToken = default(CancellationToken))
        {
            // capture the original order to be reapplied later
            var orderedKeys   = new ConcurrentQueue <TKey>();
            var orderCapturer = new TransformBlock <TInput, TInput>(
                item =>
            {
                orderedKeys.Enqueue(keyFunc(item));
                return(item);
            },
                new ExecutionDataflowBlockOptions {
                CancellationToken = cancelToken
            });

            source.LinkTo(orderCapturer, new DataflowLinkOptions {
                PropagateCompletion = true
            });

            return(new OrderedSource <TInput, TOutput, TKey>(orderedKeys, orderCapturer));
        }
Esempio n. 32
0
        static void SourceBlockTestInternal <T>(ISourceBlock <T> block)
        {
            var target = new BufferBlock <T> ();

            bool consumed;

            // invalid header
            AssertEx.Throws <ArgumentException> (
                () =>
                block.ConsumeMessage(new DataflowMessageHeader(), target, out consumed));

            // header that wasn't sent by the block doesn't throw
            block.ConsumeMessage(new DataflowMessageHeader(1), target, out consumed);

            AssertEx.Throws <ArgumentNullException> (
                () =>
                block.ConsumeMessage(new DataflowMessageHeader(1), null, out consumed));


            AssertEx.Throws <ArgumentException> (
                () =>
                block.ReserveMessage(new DataflowMessageHeader(), target));

            // header that wasn't sent by the block doesn't throw
            block.ReserveMessage(new DataflowMessageHeader(1), target);

            AssertEx.Throws <ArgumentNullException> (
                () =>
                block.ReserveMessage(new DataflowMessageHeader(1), null));

            AssertEx.Throws <ArgumentException> (
                () =>
                block.ReleaseReservation(new DataflowMessageHeader(), target));

            AssertEx.Throws <ArgumentNullException>(() => block.LinkTo(null, new DataflowLinkOptions()));

            AssertEx.Throws <ArgumentNullException>(() => block.LinkTo(target, null));
        }
Esempio n. 33
0
        //Linkd MY block to OHTER dataflow
        protected void LinkBlockToFlow <T>(ISourceBlock <T> block, IDataflow <T> otherDataflow, Predicate <T> predicate = null)
        {
            if (!IsMyChild(block))
            {
                throw new InvalidOperationException(string.Format("{0} Cannot link block to flow as the output block is not my child.", this.FullName));
            }

            if (predicate == null)
            {
                block.LinkTo(otherDataflow.InputBlock);
            }
            else
            {
                block.LinkTo(otherDataflow.InputBlock, predicate);
            }

            otherDataflow.RegisterDependency(this);

            //Make sure other dataflow also fails me
            otherDataflow.Completion.ContinueWith(otherTask =>
            {
                if (this.Completion.IsCompleted)
                {
                    return;
                }

                if (otherTask.IsFaulted)
                {
                    LogHelper.Logger.InfoFormat("{0} Downstream dataflow faulted before I am done. Fault myself.", this.FullName);
                    this.Fault(new LinkedDataflowFailedException());
                }
                else if (otherTask.IsCanceled)
                {
                    LogHelper.Logger.InfoFormat("{0} Downstream dataflow canceled before I am done. Cancel myself.", this.FullName);
                    this.Fault(new LinkedDataflowCanceledException());
                }
            });
        }
Esempio n. 34
0
        public static ISourceBlock <DecodedBlockTx> LookAhead(ISourceBlock <DecodedBlockTx> blockTxes, IDeferredChainStateCursor deferredChainStateCursor, CancellationToken cancelToken = default(CancellationToken))
        {
            // capture the original block txes order
            var orderedBlockTxes = OrderingBlock.CaptureOrder <DecodedBlockTx, DecodedBlockTx, int>(
                blockTxes, blockTx => blockTx.Index, cancelToken);

            // queue each utxo entry to be warmed up: each input's previous transaction, and each new transaction
            var queueUnspentTxLookup = InitQueueUnspentTxLookup(cancelToken);

            orderedBlockTxes.LinkTo(queueUnspentTxLookup, new DataflowLinkOptions {
                PropagateCompletion = true
            });

            // warm up each uxto entry
            var warmupUtxo = InitWarmupUtxo(deferredChainStateCursor, cancelToken);

            queueUnspentTxLookup.LinkTo(warmupUtxo, new DataflowLinkOptions {
                PropagateCompletion = true
            });

            // return the block txes with warmed utxo entries in original order
            return(orderedBlockTxes.ApplyOrder(warmupUtxo, blockTx => blockTx.Index, cancelToken));
        }
        public async Task SerializeToJson(ISourceBlock <Person> source)
        {
            using (FileStream fs = File.Create(JsonPAth))
                using (StreamWriter sw = new StreamWriter(fs))
                    using (JsonWriter jw = new JsonTextWriter(sw))
                    {
                        jw.Formatting = Formatting.Indented;
                        jw.WriteStartObject();
                        jw.WritePropertyName("People");
                        jw.WriteStartArray();
                        string temp = string.Empty;
                        while (await source.OutputAvailableAsync())
                        {
                            var person = source.Receive();
                            temp = JsonConvert.SerializeObject(person);
                            jw.WriteValue(temp);
                        }
                        jw.WriteEndArray();
                        jw.WriteEndObject();

                        //serializer.Serialize(jw, People);
                    }
        }
        internal static async Task TestReserveAndConsume <T>(
            ISourceBlock <T> block, bool reservationIsTargetSpecific = true)
        {
            bool consumed;

            block.ConsumeMessage(new DataflowMessageHeader(-99), new ActionBlock <T>(i => { }), out consumed);
            Assert.False(consumed);

            var tcs            = new TaskCompletionSource <bool>();
            var offeredMessage = default(DataflowMessageHeader);
            var target         = new DelegatePropagator <T, T>
            {
                OfferMessageDelegate = (messageHeader, value, source, consumeToAccept) =>
                {
                    offeredMessage = messageHeader;
                    tcs.TrySetResult(true);
                    return(DataflowMessageStatus.Postponed);
                }
            };

            block.LinkTo(target);
            await tcs.Task;

            Assert.True(block.ReserveMessage(offeredMessage, target));             // reserve the message

            if (reservationIsTargetSpecific)
            {
                block.ConsumeMessage(offeredMessage, new ActionBlock <T>(delegate { }), out consumed);                // different target tries to consume
                Assert.False(consumed);
            }

            block.ConsumeMessage(new DataflowMessageHeader(-99), target, out consumed);             // right target, wrong message
            Assert.False(consumed);

            block.ConsumeMessage(offeredMessage, target, out consumed);             // right target, right message
            Assert.True(consumed);
        }
Esempio n. 37
0
        private async Task StuckCompletionCannotBeCancelled(Func <CancellationToken, ISourceBlock <int> > BlockFactory, Func <ISourceBlock <int>, int> OutputCount)
        {
            CancellationTokenSource cts = new CancellationTokenSource();

            ISourceBlock <int> block  = BlockFactory(cts.Token);
            ITargetBlock <int> blockT = (ITargetBlock <int>)block;

            TestTargetBlock <int> testTarget = new TestTargetBlock <int>();

            testTarget.ConsumptionMode = DataflowMessageStatus.Declined;
            block.LinkTo(testTarget, PropagateCompletion);

            Assert.True(blockT.Post(1));
            Assert.True(blockT.Post(2));

            block.Complete();

            // Completion task should still be running
            await Task.WhenAny(block.Completion, Task.Delay(CompletionTimeout));

            Assert.False(block.Completion.IsCompleted);

            // Assumption: BufferBlock will not start target's completion until it itself completes
            await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout));

            Assert.True(testTarget.Completion.IsNotStarted());

            // Assumption: cancellation does not affect the completion of the block (unfortunately!)
            cts.Cancel();
            await Task.WhenAny(block.Completion, Task.Delay(CompletionTimeout));

            Assert.False(block.Completion.IsCompleted);
            await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout));

            Assert.True(testTarget.Completion.IsNotStarted());
            Assert.Equal(2, OutputCount(block));
        }
Esempio n. 38
0
        public static ISourceBlock <Tuple <TOut, TOut> > ForkJoinThen <TIn, TOut>
        (
            this ISourceBlock <TIn> source,
            Func <ISourceBlock <TIn>, ISourceBlock <TOut> > left,
            Func <ISourceBlock <TIn>, ISourceBlock <TOut> > right
        )
        {
            var fork =
                new BroadcastBlock <TIn>(data => data,
                                         new DataflowBlockOptions
            {
                NameFormat = "Fork {1}"
            });

            source.LinkTo(fork, LinkOptions);

            var join = new JoinBlock <TOut, TOut>();

            left(fork).LinkTo(join.Target1, LinkOptions);

            right(fork).LinkTo(join.Target2, LinkOptions);

            return(join);
        }
        public ConsulClient(string serviceName, Func <ConsulConfig> getConfig,
                            ISourceBlock <ConsulConfig> configChanged, IEnvironmentVariableProvider environmentVariableProvider,
                            ILog log, IDateTime dateTime, Func <string, AggregatingHealthStatus> getAggregatedHealthStatus)
        {
            _serviceName       = serviceName;
            _serviceNameOrigin = serviceName;

            GetConfig  = getConfig;
            _dateTime  = dateTime;
            Log        = log;
            DataCenter = environmentVariableProvider.DataCenter;

            _waitForConfigChange = new TaskCompletionSource <bool>();
            configChanged.LinkTo(new ActionBlock <ConsulConfig>(ConfigChanged));

            var address = environmentVariableProvider.ConsulAddress ?? $"{CurrentApplicationInfo.HostName}:8500";

            ConsulAddress           = new Uri($"http://{address}");
            _aggregatedHealthStatus = getAggregatedHealthStatus("ConsulClient");

            _resultChanged      = new BufferBlock <EndPointsResult>();
            _initializedVersion = new TaskCompletionSource <bool>();
            ShutdownToken       = new CancellationTokenSource();
        }
        public static async Task <int> GenerateProjectDetailPage(ISourceBlock <string> urlSource)
        {
            var errTime = 0;

            while (await urlSource.OutputAvailableAsync())
            {
                if (errTime == 10)
                {
                    return(DUPLICATED_NEWS);
                }
                var url = urlSource.Receive();
                if (MongodbChecker.CheckHeritageProjectDetailExist(url))
                {
                    errTime++;
                    continue;
                }
                var bson = GenerateHeritageProjectDetailPage(url, true);
                if (bson != null)
                {
                    MongodbSaver.SaveHeritageProjectDetail(bson);
                }
            }
            return(PROCESS_SUCCESS);
        }
 private static async Task<int> processBufferBlock(ISourceBlock<string> urlSource, Checker checker, Saver saver, BufferBlock<string> block)
 {
     var errorTime = 0;
     while (await urlSource.OutputAvailableAsync())
     {
         if(errorTime==10)
         {
             return DUPLICATED_NEWS;
         }
         var url = urlSource.Receive();
         if (checker(url))
         {
             Console.WriteLine("duplicated url: {0}", url);
             errorTime++;
             continue;
         }
         var bson = processDetailPage(url, block);
         if (bson != null)
         {
             saver(bson);
         }
     }
     return PROCESS_SUCCESS; ;
 }
 public void TopThreeMonth(ISourceBlock<List<Citizen>> source, Action<List<string>, string> PrintResult)
 {
     if (extractionCompletion == null)
     {
         ExtractData(source);
     }
     extractionCompletion.ContinueWith(
     (arg) =>
     {
         var sortedDict = from entry in globalMonthFrequency orderby entry.Value ascending select entry.Key;
         var list = sortedDict.Take(3).ToList();
         List<string> months = new List<string>();
         foreach(int month in list)
         {
             months.Add(CultureInfo.CurrentCulture.DateTimeFormat.GetMonthName(month));
         }
         PrintResult(months, "month");
     }
     );
 }
        public Task GenerateFileFingerprintsAsync(ISourceBlock<AnnotatedPath[]> annotatedPathSourceBlock,
            ITargetBlock<IFileFingerprint> fileFingerprintTargetBlock, CancellationToken cancellationToken)
        {
            var bufferBlock = new BufferBlock<IFileFingerprint>(new DataflowBlockOptions { CancellationToken = cancellationToken });

            bufferBlock.LinkTo(fileFingerprintTargetBlock, new DataflowLinkOptions { PropagateCompletion = true });

            var storeBatchBlock = new BatchBlock<IFileFingerprint>(FlushCount, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken });

            var broadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken });

            broadcastBlock.LinkTo(storeBatchBlock, new DataflowLinkOptions { PropagateCompletion = true }, ff => !ff.WasCached);
            broadcastBlock.LinkTo(bufferBlock, new DataflowLinkOptions { PropagateCompletion = true });

#if DEBUG
            var task = storeBatchBlock.Completion
                .ContinueWith(_ => Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() storeBatchBlock completed"));

            TaskCollector.Default.Add(task, "FileFingerprintManager.GenerateFileFingerprintsAsync storeBatchBlock");

            task = bufferBlock.Completion
                .ContinueWith(_ => Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() bufferBlock completed"));

            TaskCollector.Default.Add(task, "FileFingerprintManager.GenerateFileFingerprintsAsync bufferBlock");
#endif // DEBUG

            var storeTask = StoreFileFingerprintsAsync(storeBatchBlock, cancellationToken);

            var transformTask = TransformAnnotatedPathsToFileFingerprint(annotatedPathSourceBlock, broadcastBlock, cancellationToken);

            return Task.WhenAll(storeTask, transformTask);
        }
Esempio n. 44
0
 public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket bucket, ISourceBlock<Bucket> source, bool consumeToAccept)
 {
   Console.WriteLine(bucket.ToString());
   return DataflowMessageStatus.Accepted;
 }
        Task StoreFileFingerprintsAsync(ISourceBlock<IFileFingerprint[]> storeBatchBlock, CancellationToken cancellationToken)
        {
            var block = new ActionBlock<IFileFingerprint[]>(
                fileFingerprints => WriteBlobsAsync(fileFingerprints, cancellationToken),
                new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken });

            storeBatchBlock.LinkTo(block, new DataflowLinkOptions { PropagateCompletion = true });

            return block.Completion;
        }
        private void ExtractData(ISourceBlock<List<Citizen>> source)
        {
            var extractData = new ActionBlock<List<Citizen>>(
             (citizens) =>
             {
                 var today = DateTime.Today;
                 Dictionary<string, int> localNameFrequency = new Dictionary<string, int>();
                 Dictionary<int, int> localMonthFrequency = new Dictionary<int, int>();
                 foreach (Citizen citizen in citizens)
                 {

                     var bday = citizen.Birthday;
                     int age = today.Year - bday.Year;
                     if (bday > today.AddYears(-age))
                     {
                         age--;
                     }
                     Interlocked.Add(ref totalAge, age);
                     Interlocked.Increment(ref peopleCount);

                     string name = citizen.Firstname;

                     if (!localNameFrequency.ContainsKey(name))
                     {
                         localNameFrequency[name] = 0;
                     }
                     localNameFrequency[name]++;

                     int month = citizen.Birthday.Month;

                     if (!localMonthFrequency.ContainsKey(month))
                     {
                         localMonthFrequency[month] = 0;
                     }
                     localMonthFrequency[month]++;

                 }

                 mutex.WaitOne();
                 foreach (string key in localNameFrequency.Keys)
                 {
                     if (!globalNameFrequency.ContainsKey(key))
                     {
                         globalNameFrequency[key] = 0;
                     }
                     globalNameFrequency[key] += localNameFrequency[key];
                 }
                 mutex.ReleaseMutex();

                 mutex.WaitOne();
                 foreach (int key in localMonthFrequency.Keys)
                 {
                     if (!globalMonthFrequency.ContainsKey(key))
                     {
                         globalMonthFrequency[key] = 0;
                     }
                     globalMonthFrequency[key] += localMonthFrequency[key];
                 }
                 mutex.ReleaseMutex();

             }
             , new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }
             );

            source.LinkTo(extractData, new DataflowLinkOptions { PropagateCompletion = true });
            extractionCompletion = extractData.Completion;
        }
        async Task TransformAnnotatedPathsToFileFingerprint(ISourceBlock<AnnotatedPath[]> annotatedPathSourceBlock,
            ITargetBlock<IFileFingerprint> fileFingerprintTargetBlock, CancellationToken cancellationToken)
        {
            try
            {
                var targets = new ConcurrentDictionary<string, TransformBlock<AnnotatedPath, IFileFingerprint>>(StringComparer.InvariantCultureIgnoreCase);

                var routeBlock = new ActionBlock<AnnotatedPath[]>(
                    async filenames =>
                    {
                        foreach (var filename in filenames)
                        {
                            if (null == filename)
                                continue;

                            var cachedBlob = GetCachedFileFingerprint(filename.FileInfo);

                            if (null != cachedBlob)
                            {
                                await fileFingerprintTargetBlock.SendAsync(cachedBlob, cancellationToken).ConfigureAwait(false);

                                continue;
                            }

                            var host = PathUtil.GetHost(filename.FileInfo.FullName);

                            TransformBlock<AnnotatedPath, IFileFingerprint> target;
                            while (!targets.TryGetValue(host, out target))
                            {
                                target = new TransformBlock<AnnotatedPath, IFileFingerprint>(annotatedPath => ProcessFileAsync(annotatedPath.FileInfo, cancellationToken),
                                    new ExecutionDataflowBlockOptions
                                    {
                                        MaxDegreeOfParallelism = 5,
                                        CancellationToken = cancellationToken
                                    });

                                if (!targets.TryAdd(host, target))
                                    continue;

                                Debug.WriteLine($"FileFingerprintManager.GenerateBlobsAsync() starting reader for host: '{host}'");

                                target.LinkTo(fileFingerprintTargetBlock, blob => null != blob);
                                target.LinkTo(DataflowBlock.NullTarget<IFileFingerprint>());

                                break;
                            }

                            //Debug.WriteLine($"FileFingerprintManager.GenerateFileFingerprintsAsync() Sending {annotatedPath} for host '{host}'");

                            await target.SendAsync(filename, cancellationToken).ConfigureAwait(false);
                        }
                    },
                    new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 16, CancellationToken = cancellationToken });

                var distinctPaths = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);

                var distinctBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>(
                    annotatedPaths =>
                    {
                        for (var i = 0; i < annotatedPaths.Length; ++i)
                        {
                            if (!distinctPaths.Add(annotatedPaths[i].FileInfo.FullName))
                                annotatedPaths[i] = null;
                        }

                        return annotatedPaths;
                    },
                    new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, CancellationToken = cancellationToken });

                distinctBlock.LinkTo(routeBlock, new DataflowLinkOptions { PropagateCompletion = true });

                annotatedPathSourceBlock.LinkTo(distinctBlock, new DataflowLinkOptions { PropagateCompletion = true });

                await routeBlock.Completion.ConfigureAwait(false);

                foreach (var target in targets.Values)
                    target.Complete();

                await Task.WhenAll(targets.Values.Select(target => target.Completion));
            }
            catch (Exception ex)
            {
                Console.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() failed: " + ex.Message);
            }
            finally
            {
                Debug.WriteLine("FileFingerprintManager.GenerateFileFingerprintsAsync() is done");

                fileFingerprintTargetBlock.Complete();
            }
        }
Esempio n. 48
0
            // Demonstrates the consumption end of the producer and consumer pattern. 
            public async Task<Parse> AscTransConsumerAsync(ISourceBlock<Parse> source)
            {
                // counter to track the number of items that are processed
                Int64 count = 0;
                var parse = new Parse();
                var actionBlock = new ActionBlock<Parse>(
                    data =>
                    {
                        ProcessDataBuffer(data);
                        // count has to be accessed in a thread-safe manner
                        // be careful about using Interlocked,
                        // for more complicated computations, locking might be more appropriate
                        Interlocked.Increment(ref count);

                    },
                    // some small constant might be better than Unbounded, depedning on circumstances
                    new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 5}); //DataflowBlockOptions.Unbounded});
               
                source.LinkTo(actionBlock, new DataflowLinkOptions {PropagateCompletion = true});

                // this assumes source will be completed when done,
                // you need to call ascbuffer.Complete() after AscBufferProducer() for this
                await actionBlock.Completion;

                return parse;
            }
Esempio n. 49
0
        async Task ProcessMessages(ISourceBlock<Message> queue) //обрабатываю сообщения в буфере
        {
            Message message;
            while (await queue.OutputAvailableAsync())
            {
                message = (await queue.ReceiveAsync());

                User user;

                try
                {

                    using (var repository = new Repository<DutyBotDbContext>())
                    {
                        user = repository.Get<User>(u => u.TlgNumber == message.chat.id);
                        if (user == null)
                        {
                            user = new User
                            {
                                Name = message.chat.first_name,
                                TlgNumber = message.chat.id,
                                State = -1,
                                Login = "",
                                Password = "",
                                TicketNumber = ""
                            };
                            repository.Create(user);
                        }
                    }
                }
                catch (Exception)
                {
                    _bot.SendMessage(message.chat.id,
                        "Что-то пошло не так при обращении к базе данных. Дальнейшая работа не возможна.");
                    Thread.Sleep(5000);
                    return;
                }
                try
                {
                    using (var repository = new Repository<DutyBotDbContext>())
                    {
                        user = repository.Get<User>(u => u.Id == user.Id);
                        switch (user.State) //смотрю, в каком статусе пользователь 
                            //- 1 его нет, 0 ззнакомство с DUtyBot, 1 Ввод пароля, 2, проверка доступа в jira, 
                            //3 основной статус, ожидаем команды, 4 пользователь решает что делать с тикетом, который получил от бота по кнопке Проверь тикеты
                            //5 идёт мониторинг тикетов, пользователь получает уведомления, 6 пользователь решает что делать с тикетом, который получил при мониторинге
                        {
                            case -1:
                                _bot.SendMessage(message.chat.id,
                                    "Привет, " + message.chat.first_name +
                                    @"! Меня зовут DutyBot, я создан, чтобы помогать дежурить. Давай знакомиться!
 ",
                                    "{\"keyboard\": [[\"Рассказать DutyBot'у о себе\"], [\"Не хочу знакомиться, ты мне не нравишься\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                user.State = 0;
                                break;
                            case 0:
                                if ((message.text) == "Рассказать DutyBot'у о себе" |
                                    (message.text) == "Ввести учётку еще раз")
                                {
                                    _bot.SendMessage(message.chat.id,
                                        @"Чтобы мы могли работать в Jira, мне нужны твои учётные данные. Напиши мне свой логин в формате d.bot и нажми отправить.
Твои данные будут относительно безопасно храниться на сервере");
                                    user.Name = message.chat.first_name;
                                    user.TlgNumber = message.chat.id;
                                    user.State = 1;
                                    break;
                                }

                                if ((message.text) == "Не хочу знакомиться, ты мне не нравишься")
                                {
                                    _bot.SendMessage(message.chat.id,
                                        @"Очень жаль, но если надумешь, пиши. Я забуду об этом неприятном разговоре");
                                    repository.Delete(user);
                                }

                                break;
                            case 1:
                                _bot.SendMessage(message.chat.id, @"Теперь напиши пароль");

                                user.Login = message.text;
                                user.State = 2;
                                break;
                            case 2:

                                user.Password = message.text;

                                _bot.SendMessage(message.chat.id,
                                    @"Отлично, сейчас я проверю, есть ли у тебя доступ в Jira");
                                if (JiraAddFuncions.CheckConnection(_jiraConn, user.Login,
                                    user.Password))
                                {
                                    _bot.SendMessage(message.chat.id, @"Всё хорошо, можно начинать работу",
                                        "{\"keyboard\": [[\"Начнём\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");

                                    user.Password = message.text;
                                    user.State = 3;
                                    break;
                                }
                                _bot.SendMessage(message.chat.id,
                                    @"Доступа к JIra нет. Возможно учётные данные не верны. Давай попробуем ввести их еще раз. ",
                                    "{\"keyboard\": [[\"Ввести учётку еще раз\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");

                                user.State = 0;
                                break;
                            case 3:
                                switch (message.text)
                                {
                                    case "Начнём":
                                        _bot.SendMessage(message.chat.id,
                                            "Просто напиши мне сообщение, когда я тебе понадоблюсь. В ответ я пришлю меню с вариантами моих действий. Вот такое ↓",
                                            "{\"keyboard\": [[\"Кто сейчас дежурит?\"], [\"Проверь тикеты\"], [\"Помоги с дежурством\"], [\"Пока ничем\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                    case "Кто сейчас дежурит?":
                                        _bot.SendMessage(message.chat.id, DbReader.Readrespersone(),
                                            "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                    case "Проверь тикеты":
                                        _jiraConn = Jira.CreateRestClient(_jiraParam.Value,
                                            user.Login, user.Password);

                                        try
                                        {
                                            var issues =
                                                _jiraConn.GetIssuesFromJql(_filterParam.Value);
                                            IList<Issue> enumerable = issues as IList<Issue> ?? issues.ToList();
                                            if (enumerable.Any())
                                            {
                                                _ticket = enumerable.Last();

                                                user.TicketNumber = _ticket.Key.ToString();
                                                user.State = 4;
                                                _bot.SendMessage(message.chat.id, _ticket);
                                            }
                                            else
                                            {
                                                _bot.SendMessage(message.chat.id, "Тикетов нет",
                                                    "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            }

                                        }
                                        catch (Exception ex)
                                        {
                                            _bot.SendMessage(message.chat.id, "Jira не доступна",
                                                "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            var logReccord = new Log
                                            {
                                                Date = DateTime.Now,
                                                MessageTipe = "error",
                                                UserId = 0,
                                                Operation = "SendThatJiraDoesNotWork",
                                                Exception = ex.GetType() + ": " + ex.Message,
                                                AddInfo = ""
                                            };
                                            repository.Create(logReccord);
                                        }

                                        break;
                                    case "Помоги с дежурством":
                                        _bot.SendMessage(message.chat.id, "Как будем дежурить?",
                                            "{\"keyboard\": [[\"Начать мониторить тикеты\"], [\"Мониторить тикеты в моё дежурство\"], [\"Отмена\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                    case "Пока ничего":
                                        _bot.SendMessage(message.chat.id, "Ок, если что, пиши.");
                                        break;
                                    case "Начать мониторить тикеты":
                                        _bot.SendMessage(message.chat.id, @"Начинаю мониторинг.
Я буду мониторить тикеты в течение ближайших 12 часов, после чего мониторинг будет автоматически остановлен.");
                                        user.State = 5;
                                        user.DutyStart = DateTime.Now;
                                        user.DutyEnd = DateTime.Now.AddHours(12);
                                        break;
                                    case "Мониторить тикеты в моё дежурство":

                                        if (DbReader.Readuserdutystart(message.chat.id) ==
                                            Convert.ToDateTime("01.01.1900 0:00:00 ") |
                                            DbReader.Readuserdutystart(message.chat.id) ==
                                            Convert.ToDateTime("01.01.1900 0:00:00 "))
                                        {
                                            _bot.SendMessage(message.chat.id, "У тебя нет дежурств в ближайшее время",
                                                "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            break;
                                        }
                                        _bot.SendMessage(message.chat.id,
                                            "Я буду мониторить тикеты с " +
                                            DbReader.Readuserdutystart(message.chat.id).ToShortDateString() + " " +
                                            DbReader.Readuserdutystart(message.chat.id).ToShortTimeString() + " по " +
                                            DbReader.Readuserdutyend(message.chat.id).ToShortDateString() + " " +
                                            DbReader.Readuserdutyend(message.chat.id).ToShortTimeString());

                                        user.State = 5;
                                        user.DutyStart = DbReader.Readuserdutystart(message.chat.id);
                                        user.DutyEnd = DbReader.Readuserdutyend(message.chat.id);
                                        break;
                                    default:
                                        _bot.SendMessage(message.chat.id, "Чем я могу помочь?",
                                            "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                }

                                break;
                            case 4:
                                if (_ticket.Assignee == null & _ticket.Key.ToString().Equals(user.TicketNumber))
                                {
                                    switch ((message.text))
                                    {
                                        case "Распределить":
                                            _bot.SendMessage(message.chat.id, "Кому назначим?",
                                                "{\"keyboard\": [[\"Технологи\", \"Коммерция\"], [\"Админы\", \"Связисты\"], [\"Олеся\", \"Женя\"], [\"Алексей\", \"Максим\"], [\"Паша\", \"Марина\"], [\"Андрей\", \"Гриша\"], [\"Оля\", \"Настя\"], [\"Маркетинг\", \"Даша\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            break;
                                        case "Решить":
                                            JiraAddFuncions.ResolveTicket(user, _ticket, message, user.Login, _bot,
                                                _jiraConn);
                                            break;
                                        case "Назначить себе":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, user.Login, _bot,
                                                _jiraConn);
                                            break;
                                        case "Технологи":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "technologsupport",
                                                _bot,
                                                _jiraConn);
                                            break;
                                        case "Коммерция":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "crm_otdel", _bot,
                                                _jiraConn);
                                            break;
                                        case "Админы":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "Uk.Jira.TechSupport",
                                                _bot,
                                                _jiraConn);
                                            break;
                                        case "Связисты":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "uk.jira.noc", _bot,
                                                _jiraConn);
                                            break;
                                        case "Олеся":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "o.likhacheva", _bot,
                                                _jiraConn);
                                            break;
                                        case "Женя":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "ev.safonov", _bot,
                                                _jiraConn);
                                            break;
                                        case "Алексей":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "a.sapotko", _bot,
                                                _jiraConn);
                                            break;
                                        case "Максим":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "m.shemetov", _bot,
                                                _jiraConn);
                                            break;
                                        case "Андрей":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "an.zarubin", _bot,
                                                _jiraConn);
                                            break;
                                        case "Гриша":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "g.dementiev", _bot,
                                                _jiraConn);
                                            break;
                                        case "Оля":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "o.tkachenko", _bot,
                                                _jiraConn);
                                            break;
                                        case "Настя":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "a.zakharova", _bot,
                                                _jiraConn);
                                            break;
                                        case "Марина":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "m.vinnikova", _bot,
                                                _jiraConn);
                                            break;
                                        case "Паша":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "p.denisov", _bot,
                                                _jiraConn);
                                            break;
                                        case "Даша":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "d.kormushina", _bot,
                                                _jiraConn);
                                            break;
                                        case "Маркетинг":
                                            JiraAddFuncions.AssingTicket(user, _ticket, message, "Uk.Jira.TradeMarketing", _bot,
                                                _jiraConn);
                                            break;
                                        default:
                                            _bot.SendMessage(message.chat.id, "",
                                        "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            user.State = 3;
                                            break;
                                    }
                                }
                                else
                                {
                                    _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён.",
                                        "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                    user.State = 3;
                                }
                                break;
                            case 5:
                                switch (message.text)
                                {
                                    case ("Ок"):
                                        break;
                                    case ("Остановить мониторинг"):
                                        _bot.SendMessage(message.chat.id, "Готово");
                                        user.State = 3;
                                        break;
                                    case ("Продолжить мониторинг"):
                                        _bot.SendMessage(message.chat.id, "Хорошо, продолжим");
                                        break;
                                    default:
                                        _bot.SendMessage(message.chat.id, "да?",
                                            "{\"keyboard\": [[\"Остановить мониторинг\"], [\"Продолжить мониторинг\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                }
                                break;

                            case 6:
                                switch (message.text)
                                {
                                    case "Распределить":
                                        if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber))
                                        {
                                            _bot.SendMessage(message.chat.id, "Кому назначим?",
                                                "{\"keyboard\": [[\"Технологи\", \"Коммерция\"], [\"Админы\", \"Связисты\"], [\"Олеся\", \"Женя\"], [\"Алексей\", \"Максим\"], [\"Паша\", \"Марина\"], [\"Андрей\", \"Гриша\"], [\"Оля\", \"Настя\"], [\"Маркетинг\", \"Даша\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                            break;
                                        }
                                        _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён.");
                                        user.State = 5;
                                        break;
                                    case "Решить":
                                        if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber))
                                        {
                                            JiraAddFuncions.ResolveTicket(user, lastIssue, message, user.Login, _bot,
                                                _jiraConn);
                                            break;
                                        }
                                        _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён.");
                                        user.State = 5;
                                        break;
                                    case "Назначить себе":
                                        if (lastIssue.Assignee == null & lastIssue.Key.ToString().Equals(user.TicketNumber))
                                        {
                                            JiraAddFuncions.AssingTicket(user, lastIssue, message, user.Login, _bot,
                                                _jiraConn);
                                            break;
                                        }
                                        _bot.SendMessage(message.chat.id, "Похоже, тикет уже распределён.");
                                        user.State = 5;
                                        break;
                                    case "Технологи":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "technologsupport", _bot,
                                            _jiraConn);
                                        break;
                                    case "Коммерция":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "crm_otdel", _bot, _jiraConn);
                                        break;
                                    case "Админы":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "Uk.Jira.TechSupport", _bot,
                                            _jiraConn);
                                        break;
                                    case "Связисты":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "uk.jira.noc", _bot,
                                            _jiraConn);
                                        break;
                                    case "Олеся":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "o.likhacheva", _bot,
                                            _jiraConn);
                                        break;
                                    case "Женя":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "ev.safonov", _bot,
                                            _jiraConn);
                                        break;
                                    case "Алексей":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "a.sapotko", _bot, _jiraConn);
                                        break;
                                    case "Максим":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "m.shemetov", _bot,
                                            _jiraConn);
                                        break;
                                    case "Андрей":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "an.zarubin", _bot,
                                            _jiraConn);
                                        break;
                                    case "Гриша":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "g.dementiev", _bot,
                                            _jiraConn);
                                        break;
                                    case "Оля":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "o.tkachenko", _bot,
                                            _jiraConn);
                                        break;
                                    case "Настя":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "a.zakharova", _bot,
                                            _jiraConn);
                                        break;
                                    case "Марина":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "m.vinnikova", _bot,
                                            _jiraConn);
                                        break;
                                    case "Паша":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "p.denisov", _bot, _jiraConn);
                                        break;
                                    case "Маркетинг":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "Uk.Jira.TradeMarketing", _bot, _jiraConn);
                                        break;
                                    case "Даша":
                                        JiraAddFuncions.AssingTicket(user, lastIssue, message, "d.kormushina", _bot, _jiraConn);
                                        break;
                                    case "Остановить мониторинг":
                                    {
                                        user.State = 3;
                                        _bot.SendMessage(message.chat.id, "Готово");
                                        break;
                                    }

                                    case "Продолжить мониторинг":
                                    {
                                        user.State = 5;
                                        _bot.SendMessage(message.chat.id, "Хорошо, продолжим");
                                        break;
                                    }

                                    default:
                                        _bot.SendMessage(message.chat.id, "да?",
                                            "{\"keyboard\": [[\"Остановить мониторинг\"], [\"Продолжить мониторинг\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                                        break;
                                }
                                break;
                        }
                        repository.Update();
                    }
                }
                catch
                    (Exception ex)
                {
                    using (var repository = new Repository<DutyBotDbContext>())
                    {
                        var RepUser = repository.Get<User>(usr => usr.Id == user.Id);
                        if (RepUser.State > 3) RepUser.State = 3;
                        _bot.SendMessage(message.chat.id, "Что-то пошло не так при обработке сообщения. Что будем делать?", "{\"keyboard\": [[\"Проверь тикеты\"], [\"Кто сейчас дежурит?\"], [\"Помоги с дежурством\"], [\"Пока ничего\"]],\"resize_keyboard\":true,\"one_time_keyboard\":true}");
                        repository.Update();

                        var logReccord = new Log
                        {
                            Date = DateTime.Now,
                            MessageTipe = "info",
                            UserId = message.chat.id,
                            Operation = "ProcessMessage",
                            Exception = ex.Message,
                            AddInfo = ""
                        };
                        repository.Create(logReccord);
                    }
                    Thread.Sleep(5000);
                }
            }
        }
Esempio n. 50
0
        public ISourceBlock<ValidatableTx> CalculateUtxo(IChainStateCursor chainStateCursor, Chain chain, ISourceBlock<DecodedBlockTx> blockTxes, CancellationToken cancelToken = default(CancellationToken))
        {
            var chainedHeader = chain.LastBlock;
            var blockSpentTxes = new BlockSpentTxesBuilder();

            var utxoCalculator = new TransformBlock<DecodedBlockTx, ValidatableTx>(
                blockTx =>
                {
                    var tx = blockTx.Transaction;
                    var txIndex = blockTx.Index;

                    var prevTxOutputs = ImmutableArray.CreateBuilder<PrevTxOutput>(!blockTx.IsCoinbase ? tx.Inputs.Length : 0);

                    //TODO apply real coinbase rule
                    // https://github.com/bitcoin/bitcoin/blob/481d89979457d69da07edd99fba451fd42a47f5c/src/core.h#L219
                    if (!blockTx.IsCoinbase)
                    {
                        // spend each of the transaction's inputs in the utxo
                        for (var inputIndex = 0; inputIndex < tx.Inputs.Length; inputIndex++)
                        {
                            var input = tx.Inputs[inputIndex];
                            var prevTxOutput = this.Spend(chainStateCursor, txIndex, tx, inputIndex, input, chainedHeader, blockSpentTxes);

                            prevTxOutputs.Add(prevTxOutput);
                        }
                    }

                    // there exist two duplicate coinbases in the blockchain, which the design assumes to be impossible
                    // ignore the first occurrences of these duplicates so that they do not need to later be deleted from the utxo, an unsupported operation
                    // no other duplicates will occur again, it is now disallowed
                    var isDupeCoinbase = IsDupeCoinbase(chainedHeader, tx);

                    // add transaction's outputs to utxo, except for the genesis block and the duplicate coinbases
                    if (chainedHeader.Height > 0 && !isDupeCoinbase)
                    {
                        // mint the transaction's outputs in the utxo
                        this.Mint(chainStateCursor, tx, txIndex, chainedHeader);

                        // increase unspent output count
                        chainStateCursor.UnspentOutputCount += tx.Outputs.Length;

                        // increment unspent tx count
                        chainStateCursor.UnspentTxCount++;

                        chainStateCursor.TotalTxCount++;
                        chainStateCursor.TotalInputCount += tx.Inputs.Length;
                        chainStateCursor.TotalOutputCount += tx.Outputs.Length;
                    }

                    return new ValidatableTx(blockTx, chainedHeader, prevTxOutputs.MoveToImmutable());
                },
                new ExecutionDataflowBlockOptions { CancellationToken = cancelToken });

            blockTxes.LinkTo(utxoCalculator, new DataflowLinkOptions { PropagateCompletion = true });

            return OnCompleteBlock.Create(utxoCalculator, () =>
                {
                    if (!chainStateCursor.TryAddBlockSpentTxes(chainedHeader.Height, blockSpentTxes.ToImmutable()))
                        throw new ValidationException(chainedHeader.Hash);
                }, cancelToken);
        }
 private async System.Threading.Tasks.Task WriteOutputAsync(ISourceBlock<string> source)
 {
     while (await source.OutputAvailableAsync())
     {
         string data = source.Receive();
         if (OutputPane != null)
         {
             OutputPane.OutputStringThreadSafe(data);
             OutputPane.OutputStringThreadSafe("\r\n");
         }
     }
 }
Esempio n. 52
0
    public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, 
      Bucket messageValue, 
      ISourceBlock<Bucket> source, 
      bool consumeToAccept)
    {
      var lines = messageValue.ToLines();
      _bufferBlock.Post(lines);

      return DataflowMessageStatus.Accepted;
    }
Esempio n. 53
0
 public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader,
   Bucket messageValue,
   ISourceBlock<Bucket> source,
   bool consumeToAccept)
 {
   _preprocessorBlock.Post(messageValue);
   return DataflowMessageStatus.Accepted;
 }
Esempio n. 54
0
 public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, Bucket messageValue, ISourceBlock<Bucket> source, bool consumeToAccept)
 {
   messageValue.FeedTarget(_batchBlock);
   return DataflowMessageStatus.Accepted;
 }
Esempio n. 55
0
        public Task UploadBlobsAsync(AwsManager awsManager, ISourceBlock<Tuple<IFileFingerprint, AnnotatedPath>> uniqueBlobBlock,
            IReadOnlyDictionary<string, string> knowObjects, CancellationToken cancellationToken)
        {
            var blobCount = 0;
            var blobTotalSize = 0L;

            var builderBlock = new TransformBlock<Tuple<IFileFingerprint, AnnotatedPath>, S3Blobs.IUploadBlobRequest>(
                tuple =>
                {
                    string etag;
                    var exists = knowObjects.TryGetValue(tuple.Item1.Fingerprint.Key(), out etag);

                    //Debug.WriteLine($"{tuple.Item1.FullFilePath} {(exists ? "already exists" : "scheduled for upload")}");

                    if (exists)
                    {
                        // We can't check multipart uploads this way since we don't know the size
                        // of the individual parts.
                        if (etag.Contains("-"))
                        {
                            Debug.WriteLine($"{tuple.Item1.FullFilePath} is a multi-part upload with ETag {etag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}");

                            return null;
                        }

                        var expectedETag = tuple.Item1.Fingerprint.S3ETag();

                        if (string.Equals(expectedETag, etag, StringComparison.InvariantCultureIgnoreCase))
                            return null;

                        Console.WriteLine($"ERROR: {tuple.Item1.FullFilePath} tag mismatch {etag}, expected {expectedETag} {tuple.Item1.Fingerprint.Key().Substring(0, 12)}");
                    }

                    var request = awsManager.BuildUploadBlobRequest(tuple);

                    if (null == request)
                        return null;

                    Interlocked.Increment(ref blobCount);

                    Interlocked.Add(ref blobTotalSize, request.FileFingerprint.Fingerprint.Size);

                    return request;
                },
                new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });

            var uploader = new ActionBlock<S3Blobs.IUploadBlobRequest>(
                blob => UploadBlobAsync(awsManager, blob, cancellationToken),
                new ExecutionDataflowBlockOptions
                {
                    MaxDegreeOfParallelism = 4,
                    CancellationToken = cancellationToken
                });

            builderBlock.LinkTo(uploader, new DataflowLinkOptions { PropagateCompletion = true }, r => null != r);
            builderBlock.LinkTo(DataflowBlock.NullTarget<S3Blobs.IUploadBlobRequest>());

            uniqueBlobBlock.LinkTo(builderBlock, new DataflowLinkOptions { PropagateCompletion = true });

            var tasks = new List<Task>();

#if DEBUG
            var uploadDoneTask = uploader.Completion
                .ContinueWith(
                    _ => Debug.WriteLine($"Done uploading blobs: {blobCount} items {SizeConversion.BytesToGiB(blobTotalSize):F2}GiB"), cancellationToken);

            tasks.Add(uploadDoneTask);
#endif

            tasks.Add(uploader.Completion);

            return Task.WhenAll(tasks);
        }
Esempio n. 56
0
 /// <summary>
 /// Initializes a new instance of the <see cref="ArrayDataReader" /> class.
 /// </summary>
 /// <param name="sourceBlock">The source block.</param>
 /// <param name="fieldCount">The field count.</param>
 public ArrayDataReader(ISourceBlock<object> sourceBlock, int fieldCount)
 {
     _sourceBlock = sourceBlock;
     FieldCount = fieldCount;
 }
Esempio n. 57
0
 public AsyncWordVectorLoader(string filename) : base(filename)
 {
     _linesSource = ReadWordVecLines(filename);
 }
Esempio n. 58
0
 /// <summary>
 /// Offers the message.
 /// </summary>
 /// <param name="messageHeader">The message header.</param>
 /// <param name="messageValue">The message value.</param>
 /// <param name="source">The source.</param>
 /// <param name="consumeToAccept">The consume to accept.</param>
 /// <returns>DataflowMessageStatus.</returns>
 public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, LogEntry messageValue, ISourceBlock<LogEntry> source, Boolean consumeToAccept)
 {
     return _BatchBlock.OfferMessage(messageHeader, messageValue, source, consumeToAccept);
 }