コード例 #1
0
        public async Task Save(String storageConnectionString, IProcess process, LeaseManager leaseManager)
        {
            process.SaveState(out var serialized, out var label);

            var cloudBlobContainer = await AzureBlobStorageStateManager.GetCloudBlobContainer(storageConnectionString, HostLogger, initialize : false);

            await AzureBlobStorageStateManager
            .Save(cloudBlobContainer, deploymentId, HostLogger, processId,
                  new ProcessState(deploymentTimestamp, lastCheckpointedSequenceNumber, serialized, seenClock, ourClock), leaseManager.LeaseId);
        }
コード例 #2
0
        public static async Task Doorbell(Microsoft.Azure.WebJobs.ExecutionContext executionContext, ILogger logger, EventData[] messages)
        {
            var stopwatch = new System.Diagnostics.Stopwatch();

            stopwatch.Start();

            var msg = DoorbellMessage.Deserialize(messages[0].Body.Array);

            var processId = msg.ProcessId;

            var applicationInfo = new TStaticApplicationInfo();

            var configuration = applicationInfo.GetHostConfiguration();

            var hostlogger = new LoggerWrapper(logger, $"[p{processId:d3} doorbell] ");

            var lastRing = messages[messages.Length - 1].SystemProperties.SequenceNumber;


            try
            {
                var cloudBlobContainer = await AzureBlobStorageStateManager.GetCloudBlobContainer(configuration.StorageConnectionString, hostlogger);

                var stateBlob = cloudBlobContainer.GetBlockBlobReference(AzureBlobStorageStateManager.BlobName(processId));

                var leaseManager = new LeaseManager(stateBlob);

                if (await leaseManager.TryGetLease(hostlogger))
                {
                    hostlogger.LogInformation($"{DateTime.UtcNow:o} Rings x{messages.Length} on {Environment.MachineName}, through #{lastRing}, lease acquired");
                    await RunHost(applicationInfo, configuration, processId, hostlogger, logger, leaseManager, stopwatch, executionContext.InvocationId);
                }
                else
                {
                    hostlogger.LogInformation($"{DateTime.UtcNow:o} Rings x{messages.Length} on {Environment.MachineName}, through #{lastRing}, ignored");
                }
            }
            catch (Exception e)
            {
                hostlogger.LogError($"Doorbell failed: {e}");
            }
        }
コード例 #3
0
        private static async Task RunHost(IStaticApplicationInfo applicationInfo, FunctionsHostConfiguration configuration, uint processId,
                                          ILogger hostlogger, ILogger logger, LeaseManager leaseManager, System.Diagnostics.Stopwatch stopwatch, Guid invocationId)
        {
            try
            {
                var host = new Host <TStaticApplicationInfo>(applicationInfo, configuration, logger, processId, stopwatch, invocationId);

                var done = await host.ResumeFromCheckpoint(leaseManager);

                // if we get here, there is no more work for this process
                await leaseManager.Release();

                hostlogger.LogInformation($"Lease released");

                if (!done)
                {
                    await host.RingMyself();
                }
                else
                {
                    await host.FinalRecheck();
                }

                await host.Cleanup(false);
            }
            catch (Exception e)
            {
                hostlogger.LogError($"RunHost Failed: {e}");

                // throw exception, so event hub delivers doorbell message again
                // TODO think about poison messages
                throw;
            }
            finally
            {
                hostlogger.LogDebug($"Control returned");
            }
        }
コード例 #4
0
        public async Task <bool> ResumeFromCheckpoint(LeaseManager leaseManager)
        {
            var start = stopwatch.Elapsed;

            try
            {
                // kick off the load
                var storageConnectionString = configuration.StorageConnectionString;
                var loadtask = AzureBlobStorageStateManager.Load(storageConnectionString, HostLogger, processId);

                // register host services
                application.HostServices.RegisterSend(Send);
                application.HostServices.RegisterApplicationLogger(ApplicationLogger);
                application.HostServices.RegisterRuntimeLogger(RuntimeLogger);

                //          if (_configuration.AppInsightsInstrumentationKey != null)
                //                _application.HostServices.RegisterTelemetryListener(new ApplicationInsightsTelemetryListener(_configuration.AppInsightsInstrumentationKey, "eventhubs"));

                if (blobTelemetryListener != null)
                {
                    application.HostServices.RegisterTelemetryListener(blobTelemetryListener);
                }

                // read the checkpoint
                var checkpointedState = await loadtask;

                SetDeploymentTimestamp(checkpointedState.DeploymentTimestamp);
                seenClock = checkpointedState.SeenClock;
                ourClock  = checkpointedState.OurClock;
                var state = checkpointedState.State;
                lastCheckpointedSequenceNumber = checkpointedState.Version;

                HostLogger.LogDebug($"Resuming at position {lastCheckpointedSequenceNumber}");

                Connections.ResumeFrom(lastCheckpointedSequenceNumber);

                // build the process

                IProcess process = application.MakeProcess(processId);

                if (state == null)
                {
                    process.FirstStart();
                }
                else
                {
                    process.Restore(state, out var label);
                }

                process.BecomePrimary();

                // start the message receiving loop

                long lastProcessedPosition = 0;

                int iterationCount = 0;
                int dedupCount     = 0;
                int receiveCount   = 0;
                int loopbackCount  = 0;
                int clientCount    = 0;

                TimeSpan lastReport           = stopwatch.Elapsed;
                long     lastReportedPosition = 0;

                while (true)
                {
                    iterationCount++;


                    if (lastProcessedPosition > lastReportedPosition && stopwatch.Elapsed - lastReport > TimeSpan.FromSeconds(15))
                    {
                        HostLogger.LogInformation($"progress to v{lastProcessedPosition} after {stopwatch.Elapsed.TotalSeconds:f2}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
                        lastReportedPosition = lastProcessedPosition;
                        lastReport           = stopwatch.Elapsed;
                        CombinedLogger.Flush();
                    }

                    try
                    {
                        bool outOfTime = stopwatch.Elapsed > configuration.TimeLimit;

                        IEnumerable <EventData> eventData = outOfTime ? null :
                                                            await Connections.ProcessReceiver.ReceiveAsync(configuration.MaxReceiveBatchSize, iterationCount == 1?TimeSpan.FromSeconds(10) : configuration.ReceiveWaitTime);

                        if (eventData == null)
                        {
                            HostLogger.LogTrace($"{ DateTime.UtcNow:o} Received nothing. {Connections.ProcessReceiver.RuntimeInfo.LastSequenceNumber}");

                            if (process.RequestsOutstanding() && !outOfTime)
                            {
                                HostLogger.LogDebug($"continue for outstanding requests.");
                                CombinedLogger.Flush();
                                continue;
                            }
                            else if (lastProcessedPosition > lastCheckpointedSequenceNumber)
                            {
                                await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());

                                lastCheckpointedSequenceNumber = lastProcessedPosition;
                                await Save(storageConnectionString, process, leaseManager);

                                HostLogger.LogDebug($"continue for saving snapshot.");
                                continue;
                            }
                            else
                            {
                                // no more work to do here
                                HostLogger.LogInformation($"{(outOfTime ? "out of time" : "done")} after {stopwatch.Elapsed.TotalSeconds}s, {receiveCount + dedupCount + loopbackCount + clientCount} messages ({receiveCount} new, {loopbackCount} loopback, {clientCount} client, {dedupCount} deduped) in {iterationCount} batches");
                                CombinedLogger.Flush();
                                return(!outOfTime);
                            }
                        }

                        foreach (var ed in eventData)
                        {
                            var body    = ed.Body;
                            var message = ProcessMessage.Deserialize(body.Array);

                            HostLogger.LogTrace($"{DateTime.UtcNow:o} Received {message}");

                            if (!message.IsExternalRequest && message.DeploymentTimestamp < deploymentTimestamp)
                            {
                                HostLogger.LogDebug($"Ignoring message from earlier deployment {message.DeploymentTimestamp}");
                            }
                            else if (!message.IsExternalRequest && message.DeploymentTimestamp > deploymentTimestamp)
                            {
                                HostLogger.LogError($"****** MISSING STATE ERROR process state is from older deployment, should have been initialized for new one! {message.DeploymentTimestamp} != {deploymentTimestamp}");
                            }
                            else if (message.IsSequenced &&
                                     seenClock.HasSeen(message.Source, message.LastClock.Value))
                            {
                                dedupCount++;
                                HostLogger.LogTrace($"Deduping: {message}");
                            }
                            else if (message.IsExternalRequest)
                            {
                                clientCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <IMessage> payload;
                                MemoryStream    stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var m in payload)
                                {
                                    process.ProcessMessage(m);
                                }
                            }
                            else if (message.IsLoopback)
                            {
                                loopbackCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <IMessage> payload;
                                MemoryStream    stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <IMessage>)payloadSerializerLoopback.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var m in payload)
                                {
                                    process.ProcessMessage(m);
                                }
                            }
                            else
                            {
                                receiveCount++;
                                HostLogger.LogTrace($"Processing: {message}");

                                // deserialize content
                                List <KeyValuePair <long, IMessage> > payload;
                                MemoryStream stream = new MemoryStream(message.Payload);
                                using (var binaryDictionaryReader = XmlDictionaryReader.CreateBinaryReader(stream, XmlDictionaryReaderQuotas.Max))
                                {
                                    payload = (List <KeyValuePair <long, IMessage> >)payloadSerializer.ReadObject(binaryDictionaryReader);
                                }

                                // Iterate
                                foreach (var kvp in payload)
                                {
                                    if (!seenClock.HasSeen(message.Source, kvp.Key))
                                    {
                                        process.ProcessMessage(kvp.Value);
                                    }
                                }

                                // Update seen clock.
                                seenClock.Set(message.Source, message.LastClock.Value);
                            }

                            lastProcessedPosition = ed.SystemProperties.SequenceNumber;

                            // Checkpoint and commit every X.
                            if (lastProcessedPosition > lastCheckpointedSequenceNumber &&
                                lastProcessedPosition % configuration.CheckpointInterval == configuration.CheckpointInterval - 1)
                            {
                                await Task.WhenAll(Senders.Select(s => s.WaitForCurrentWorkToBeServiced()).ToList());

                                // Checkpoint state.
                                lastCheckpointedSequenceNumber = lastProcessedPosition;
                                await Save(storageConnectionString, process, leaseManager);
                            }
                        }
                    }
                    catch (System.OperationCanceledException)
                    {
                        HostLogger.LogDebug($"receive returned OperationCanceledException, scheduling retry!");
                        continue;
                    }
                }
            }
            finally
            {
                if (this.collectHostEvents)
                {
                    this.blobTelemetryListener.OnApplicationEvent(processId, invocationId.ToString(), $"Process {processId}", "", OperationSide.Caller, OperationType.Host, (stopwatch.Elapsed - start).TotalMilliseconds);
                }
            }
        }