Exemplo n.º 1
0
        public override async Task Execute()
        {
            IReceive arv = null;

            if (Receivers?.Count == 1)
            {
                arv = Receivers[0];
            }
            if (arv == null)
            {
                if (AllReceiversAsync)
                {
                    arv = new AsyncReceiverMultiple(Receivers.Select(it => it.Value).ToArray());
                }
                else
                {
                    arv = new SyncReceiverMultiple(Receivers.Select(it => it.Value).ToArray());
                }
            }
            await arv.LoadData();

            bool existsVar = (RuntimeParameters?.Length ?? 0) > 0;

            string[] nameObjectsWithVariables = null;
            if (existsVar)
            {
                nameObjectsWithVariables = RuntimeParameters
                                           .SelectMany(it => it.NameObjectsToApplyTo)
                                           .Select(it => it.NameObjectToApplyTo.ToLowerInvariant())
                                           .Distinct()
                                           .ToArray();
            }
            IRow[] data = arv.valuesRead;
            foreach (var filterKV in FiltersAndTransformers)
            {
                var var = filterKV.Value as TransformIntoVariable;
                if (var != null)
                {
                    if (!existsVar)
                    {
                        //TODO:log
                        continue;
                    }
                    var param = RuntimeParameters.FirstOrDefault(it => it.VariableName == var.VariableName);
                    if (param == null)
                    {
                        throw new ArgumentException($"in runtime parameters I cannot find variable {var.VariableName}");
                    }
                    await var.Run();

                    variables[param] = var.Result;
                    continue;
                }
                bool hasVar = (nameObjectsWithVariables?.Length > 0) && (nameObjectsWithVariables.Contains(filterKV.Value.Name.ToLowerInvariant()));
                if (hasVar)
                {
                    TransformPropertyFromVar(filterKV.Value);
                }
                //TODO: see also IFilterTransformer
                var transform = filterKV.Value as ITransform;
                if (transform != null)
                {
                    transform.valuesRead = data;
                    await transform.Run();

                    data = transform.valuesTransformed;
                    continue;
                }
                var filter = filterKV.Value as IFilter;
                if (filter != null)
                {
                    filter.valuesRead = data;
                    await filter.Run();

                    data = filter.valuesTransformed;
                    continue;
                }
                Debug.Assert(false, "base object is not found");
                //@class.Log(LogLevel.Error,0,$"base object is not found",null,null);
                Debug.Assert(false, "filter is not found");
            }
            //await SenderData(data);
            if (Senders.Count == 0)
            {
                return;
            }

            ISend send = null;

            if (Senders.Count == 1)
            {
                send = Senders[0];
            }
            if (send == null)
            {
                if (AllSendersAsync)
                {
                    send = new ASyncSenderMultiple(Senders.Select(it => it.Value).ToArray());
                }
                else
                {
                    send = new SyncSenderMultiple(Senders.Select(it => it.Value).ToArray());
                }
            }
            send.valuesToBeSent = data;
            await send.Send();
        }
Exemplo n.º 2
0
        public async Task <bool> ResumeFromCheckpoint(LeaseManager leaseManager)
        {
            var start = stopwatch.Elapsed;

            try
            {
                // kick off the load
                var storageConnectionString = configuration.StorageConnectionString;
                var loadtask = AzureBlobStorageStateManager.Load(storageConnectionString, HostLogger, processId);

                // register host services
                application.HostServices.RegisterSend(Send);
                application.HostServices.RegisterApplicationLogger(ApplicationLogger);
                application.HostServices.RegisterRuntimeLogger(RuntimeLogger);

                //          if (_configuration.AppInsightsInstrumentationKey != null)
                //                _application.HostServices.RegisterTelemetryListener(new ApplicationInsightsTelemetryListener(_configuration.AppInsightsInstrumentationKey, "eventhubs"));

                if (blobTelemetryListener != null)
                {
                    application.HostServices.RegisterTelemetryListener(blobTelemetryListener);
                }

                // read the checkpoint
                var checkpointedState = await loadtask;

                SetDeploymentTimestamp(checkpointedState.DeploymentTimestamp);
                seenClock = checkpointedState.SeenClock;
                ourClock  = checkpointedState.OurClock;
                var state = checkpointedState.State;
                lastCheckpointedSequenceNumber = checkpointedState.Version;

                HostLogger.LogDebug($"Resuming at position {lastCheckpointedSequenceNumber}");

                Connections.ResumeFrom(lastCheckpointedSequenceNumber);

                // build the process

                IProcess process = application.MakeProcess(processId);

                if (state == null)
                {
                    process.FirstStart();
                }
                else
                {
                    process.Restore(state, out var label);
                }

                process.BecomePrimary();

                // start the message receiving loop

                long lastProcessedPosition = 0;

                int iterationCount = 0;
                int dedupCount     = 0;
                int receiveCount   = 0;
                int loopbackCount  = 0;
                int clientCount    = 0;

                TimeSpan lastReport           = stopwatch.Elapsed;
                long     lastReportedPosition = 0;

                while (true)
                {
                    iterationCount++;


                    if (lastProcessedPosition > lastReportedPosition && stopwatch.Elapsed - lastReport > TimeSpan.FromSeconds(15))
                    {
                        HostLogger.LogInformation($"progress to v{lastProcessedPosition} after {stopwatch.Elapsed.TotalSeconds:f2}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
                        lastReportedPosition = lastProcessedPosition;
                        lastReport           = stopwatch.Elapsed;
                        CombinedLogger.Flush();
                    }

                    try
                    {
                        bool outOfTime = stopwatch.Elapsed > configuration.TimeLimit;

                        IEnumerable <EventData> eventData = outOfTime ? null :
                                                            await Connections.ProcessReceiver.ReceiveAsync(configuration.MaxReceiveBatchSize, iterationCount == 1?TimeSpan.FromSeconds(10) : configuration.ReceiveWaitTime);

                        if (eventData == null)
                        {
                            HostLogger.LogTrace($"{ DateTime.UtcNow:o} Received nothing. {Connections.ProcessReceiver.RuntimeInfo.LastSequenceNumber}");

                            if (process.RequestsOutstanding() && !outOfTime)
                            {
                                HostLogger.LogDebug($"continue for outstanding requests.");
                                CombinedLogger.Flush();
                                continue;
                            }
                            else if (lastProcessedPosition > lastCheckpointedSequenceNumber)
                            {
                                await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());

                                lastCheckpointedSequenceNumber = lastProcessedPosition;
                                await Save(storageConnectionString, process, leaseManager);

                                HostLogger.LogDebug($"continue for saving snapshot.");
                                continue;
                            }
                            else
                            {
                                // no more work to do here
                                HostLogger.LogInformation($"{(outOfTime ? "out of time" : "done")} after {stopwatch.Elapsed.TotalSeconds}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
                                CombinedLogger.Flush();
                                return(!outOfTime);
                            }
                        }

                        foreach (var ed in eventData)
                        {
                            var body    = ed.Body;
                            var message = ProcessMessage.Deserialize(body.Array);

                            HostLogger.LogTrace($"{DateTime.UtcNow:o} Received {message}");

                            if (!message.IsExternalRequest && message.DeploymentTimestamp < deploymentTimestamp)
                            {
                                HostLogger.LogDebug($"Ignoring message from earlier deployment {message.DeploymentTimestamp}");
                            }
                            else if (!message.IsExternalRequest && message.DeploymentTimestamp > deploymentTimestamp)
                            {
                                HostLogger.LogError($"****** MISSING STATE ERROR process state is from older deployment, should have been initialized for new one! {message.DeploymentTimestamp} != {deploymentTimestamp}");
                            }
                            else if (message.IsSequenced &&
                                     seenClock.HasSeen(message.Source, message.LastClock.Value))
                            {
                                dedupCount++;
                                HostLogger.LogTrace($"Deduping: {message}");
                            }
                            else if (message.IsExternalRequest)
                            {
                                clientCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <IMessage> payload;
                                MemoryStream    stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var m in payload)
                                {
                                    process.ProcessMessage(m);
                                }
                            }
                            else if (message.IsLoopback)
                            {
                                loopbackCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <IMessage> payload;
                                MemoryStream    stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var m in payload)
                                {
                                    process.ProcessMessage(m);
                                }
                            }
                            else
                            {
                                receiveCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <KeyValuePair <long, IMessage> > payload;
                                MemoryStream stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <KeyValuePair <long, IMessage> >)payloadSerializer.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var kvp in payload)
                                {
                                    if (!seenClock.HasSeen(message.Source, kvp.Key))
                                    {
                                        process.ProcessMessage(kvp.Value);
                                    }
                                }

                                // Update seen clock.
                                seenClock.Set(message.Source, message.LastClock.Value);
                            }

                            lastProcessedPosition = ed.SystemProperties.SequenceNumber;

                            // Checkpoint and commit every X.
                            if (lastProcessedPosition > lastCheckpointedSequenceNumber &&
                                lastProcessedPosition % configuration.CheckpointInterval == configuration.CheckpointInterval - 1)
                            {
                                await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());

                                // Checkpoint state.
                                lastCheckpointedSequenceNumber = lastProcessedPosition;
                                await Save(storageConnectionString, process, leaseManager);
                            }
                        }
                    }
                    catch (System.OperationCanceledException)
                    {
                        HostLogger.LogDebug($"receive returned OperationCanceledException, scheduling retry!");
                        continue;
                    }
                }
            }
            finally
            {
                if (this.collectHostEvents)
                {
                    this.blobTelemetryListener.OnApplicationEvent(processId, invocationId.ToString(), $"Process {processId}", "", OperationSide.Caller, OperationType.Host, (stopwatch.Elapsed - start).TotalMilliseconds);
                }
            }
        }