Ejemplo n.º 1
0
        /// <summary>
        /// This method fails or times out with at least 20% chance
        /// </summary>
        private Task CallUnpredictableExternalAPI(SQSEvent.SQSMessage message, ILambdaLogger contextLogger)
        {
            contextLogger.Log(" ** inside CallExternalService **");

            //about 10% of requests should explode
            var seed = Guid.NewGuid().GetHashCode();

            contextLogger.Log($" ** checking RND seed {seed} **");
            var rnd           = new Random(seed);
            int failureChance = rnd.Next(1, 11);

            if (failureChance == 5)
            {
                contextLogger.Log($" ** about to throw exception, failureChance = {failureChance} **");
                throw new ApplicationException();
            }

            //about 10% of requests should time out
            failureChance = rnd.Next(1, 11);
            if (failureChance == 10)
            {
                contextLogger.Log($" ** about to freeze,  failureChance = {failureChance} **");
                Thread.Sleep(TimeOutSeconds * 1000);
                return(Task.Run(() => {}));
            }

            //this is happy path
            return(this.ResendMessageToAnotherQueue(OutputQueueUrl, message, contextLogger));
        }
Ejemplo n.º 2
0
        public async Task FunctionHandler(ILambdaContext context)
        {
            _context = context;
            _logger  = _context.Logger;
            _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started");
            _verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true);
            var maxTime   = 60 * 1000;
            var rateLimit = 20 * 1000;
            var sw        = Stopwatch.StartNew();

            long remaining = 0;

            do
            {
                _logger.Log($"New Execution Started, currently elpased: {sw.ElapsedMilliseconds}");
                var elapsed = await Execute();

                var rateAwait = rateLimit - elapsed;

                remaining = maxTime - (sw.ElapsedMilliseconds + Math.Max(rateLimit, 2 * elapsed) + 1000);

                if (rateAwait > 0 && remaining > 0)
                {
                    Log($"Rate Limit Awaiting {rateAwait} [ms] remaining out of {rateLimit} [ms]. Total elapsed: {sw.ElapsedMilliseconds} [ms]");
                    await Task.Delay((int)rateAwait);
                }

                await Task.Delay(1000);
            } while (remaining > 0);
        }
Ejemplo n.º 3
0
        public async Task FunctionHandler(ILambdaContext context)
        {
            var sw = Stopwatch.StartNew();

            _context = context;
            _logger  = _context.Logger;
            _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started");

            _verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true);
            var githubToken = Environment.GetEnvironmentVariable("github_token");

            if (githubToken.IsNullOrEmpty())
            {
                throw new Exception("Environment Variable 'github-token' was not defined!");
            }

            if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false))
            {
                Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available.");
            }

            string accessToken;

            if (githubToken.IsHex())
            {
                Log($"Property 'githubToken' was determined to be a hexadecimal and will be used as accessToken");
                accessToken = githubToken;
            }
            else
            {
                Log($"Fetching github access token '{githubToken ?? "undefined"}' from secrets manager.");
                accessToken = (await _SM.GetSecret(githubToken)).JsonDeserialize <AmazonSecretsToken>()?.token;
            }

            var rateLimit = Environment.GetEnvironmentVariable("github_rate_limit").ToIntOrDefault(1000);

            _GIT = new GitHubHelper(new GITWrapper.GitHub.Models.GitHubRepoConfig(accessToken: accessToken, maxRatePerHour: rateLimit)
            {
                user       = Environment.GetEnvironmentVariable("github_user"),
                branch     = Environment.GetEnvironmentVariable("github_branch"),
                repository = Environment.GetEnvironmentVariable("github_repository"),
                userAgent  = Environment.GetEnvironmentVariable("user_agent") ?? "Asmodat Launcher Toolkit",
            });

            try
            {
                Log($"Processing...");
                await Processing();
            }
            finally
            {
                _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {sw.ElapsedMilliseconds} [ms]");
                await Task.Delay(2500);
            }
        }
Ejemplo n.º 4
0
        private async Task ChangeVisibilityTimeOutAsyncTask(SQSEvent.SQSMessage message, ILambdaLogger contextLogger, int timeoutSeconds)
        {
            contextLogger.Log($" *** about to change visibility timeout  to {timeoutSeconds}  seconds***");

            var batRequest = new ChangeMessageVisibilityRequest
            {
                ReceiptHandle     = message.ReceiptHandle,
                VisibilityTimeout = timeoutSeconds,
                QueueUrl          = InputQueueUrl
            };

            var response = await this.SqSClient.ChangeMessageVisibilityAsync(batRequest);

            contextLogger.Log($" *** visibility timeout result status code is {response.HttpStatusCode} ***");
        }
Ejemplo n.º 5
0
        private async Task <CloudTrailRecords> ExtractCloudTrailRecordsAsync(ILambdaLogger logger, byte[] input)
        {
            var appearsGzipped = ResponseAppearsGzipped(input);

            logger.LogLine($"Input appears to be gzipped: {appearsGzipped}");
            if (appearsGzipped)
            {
                using (var contents = new MemoryStream())
                    using (var gz = new GZipStream(new MemoryStream(input), CompressionMode.Decompress))
                    {
                        await gz.CopyToAsync(contents);

                        input = contents.ToArray();
                    }
            }

            var serializedRecords = Encoding.UTF8.GetString(input);

            logger.Log(serializedRecords);
            return(JsonConvert.DeserializeObject <CloudTrailRecords>(serializedRecords));

            bool ResponseAppearsGzipped(byte[] bytes)
            {
                var header = new byte[GZipHeaderBytes.Length];

                Array.Copy(bytes, header, header.Length);
                return(header.SequenceEqual(GZipHeaderBytes));
            }
        }
Ejemplo n.º 6
0
 public override void Write(string value)
 {
     _logger.Log(value);
     if (this._writeToInnerWriter)
     {
         this._innerWriter?.Write(value);
     }
 }
Ejemplo n.º 7
0
        private void Log(string msg)
        {
            if (msg.IsNullOrEmpty() || !_verbose)
            {
                return;
            }

            _logger.Log(msg);
        }
Ejemplo n.º 8
0
        public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
        {
            string message = string.Empty;

            if (formatter != null)
            {
                message = formatter(state, exception);
            }

            lambdaLogger.Log(message);
        }
Ejemplo n.º 9
0
 public void Send(DataPointUploadMessage msg)
 {
     try
     {
         var request = _requestor.GetRequestor();
         using (var rs = request.GetWriteStream())
         {
             Serializer.Serialize(rs, msg);
             // flush the message before disposing
             rs.Flush();
         }
         try
         {
             using (request.Send())
             {
             }
         }
         catch (SecurityException)
         {
             log.Log("API token for sending metrics to SignalFx is invalid");
         }
     }
     catch (Exception ex)
     {
         if (ex is WebException)
         {
             var webex = ex as WebException;
             using (var exresp = webex.Response)
             {
                 if (exresp != null)
                 {
                     var stream2  = exresp.GetResponseStream();
                     var reader2  = new StreamReader(stream2);
                     var errorStr = reader2.ReadToEnd();
                     log.Log(errorStr);
                 }
             }
         }
     }
 }
Ejemplo n.º 10
0
        private void InternalLog(LogMessage logMessage)
        {
            var options = new JsonSerializerOptions
            {
                WriteIndented    = true,
                IgnoreNullValues = true
            };

            options.Converters.Add(new JsonStringEnumConverter());
            options.Converters.Add(new LogMessageJsonConverter());

            var jsonString = JsonSerializer.Serialize(logMessage, options);

            lambdaLogger.Log(jsonString);
        }
Ejemplo n.º 11
0
        private SkillResponse HandleNextEventRequest(IntentRequest intentRequest, ILambdaLogger logger)
        {
            string message;

            if (intentRequest.Intent.Slots.TryGetValue("calendar_name", out var calendarNameSlot) &&
                TryParseCalendarName(calendarNameSlot, out CalendarType? calendarType, logger) &&
                calendarType != CalendarType.None)
            {
                Calendar calendar = CalendarService.GetCalendar(calendarType.Value)
                                    .ConfigureAwait(false).GetAwaiter().GetResult();

                CalendarEntry nextEvent = CalendarService.GetFutureCalendarEntries(calendar)
                                          .FirstOrDefault();

                if (nextEvent != null)
                {
                    string ssml = CreateSsmlForEvent(calendar, nextEvent);

                    logger.Log($"Created response SSML: {ssml}");

                    return(ResponseBuilder.Tell(new SsmlOutputSpeech
                    {
                        Ssml = ssml
                    }));
                }
                else
                {
                    message = $"I'm sorry. I couldn't find any future calendar entries in the {calendar.Name} calendar.";
                }
            }
            else
            {
                message = "I'm sorry. I wasn't able to understand that calendar name.";
            }

            var response = ResponseBuilder.Tell(new PlainTextOutputSpeech
            {
                Text = message
            });

            response.Response.ShouldEndSession = true;

            return(response);
        }
Ejemplo n.º 12
0
        private bool TryParseCalendarName(Slot calendarNameSlot, out CalendarType?calendarType, ILambdaLogger logger)
        {
            string calendarTypeText          = calendarNameSlot.Value;
            var    firstResolution           = calendarNameSlot.Resolution?.Authorities?.FirstOrDefault();
            var    firstResolutionStatusCode = firstResolution?.Status?.Code;
            var    firstResolutionValue      = firstResolution?.Values?.FirstOrDefault()?.Value?.Name;

            if (firstResolutionStatusCode == "ER_SUCCESS_MATCH" &&
                !String.IsNullOrEmpty(firstResolutionValue))
            {
                calendarTypeText = firstResolutionValue;
            }

            logger.Log($"Received request for calendar: {calendarNameSlot.Value} - Resolution code: {firstResolutionStatusCode} - Resolution value: {firstResolutionValue} - Resolved to: {calendarTypeText}");

            if (Enum.TryParse(typeof(CalendarType), calendarTypeText, out var tempCalendarType))
            {
                calendarType = (CalendarType)tempCalendarType;
                return(true);
            }

            calendarType = null;
            return(false);
        }
Ejemplo n.º 13
0
        private async Task <SendMessageResponse> ResendMessageToAnotherQueue(string queue, SQSEvent.SQSMessage message, ILambdaLogger logger, int delaySeconds = 0)
        {
            logger.Log($" ** sending message {message.Body} to  queue {queue} ** ");
            var writeMessageRequest = new SendMessageRequest(queue, message.Body);

            int retryCounter = 0;

            if (message.MessageAttributes.ContainsKey(RetryCount))
            {
                retryCounter = Convert.ToInt32(message.MessageAttributes[RetryCount].StringValue);
                retryCounter++;
            }

            writeMessageRequest.MessageAttributes = new Dictionary <string, MessageAttributeValue>();
            writeMessageRequest.MessageAttributes.Add(RetryCount, new MessageAttributeValue()
            {
                DataType = "String", StringValue = (retryCounter).ToString()
            });

            //Normalize distribution of incoming messages in time by function x2
            writeMessageRequest.DelaySeconds = retryCounter * retryCounter * delaySeconds;

            return(await this.SqSClient.SendMessageAsync(writeMessageRequest));
        }
Ejemplo n.º 14
0
        public async Task <Document> CreateOrder(string orderStr)
        {
            try
            {
                _logger.Log($"Raw data : {orderStr}");

                var config = new AmazonDynamoDBConfig();
                config.RegionEndpoint = RegionEndpoint.APSoutheast2;
                _client = new AmazonDynamoDBClient(config);

                var finalText = orderStr.Trim().Replace(@"\", "");
                _logger.Log($"After replace text: {finalText}");

                var x = finalText.Substring(1, finalText.Length - 2);
                _logger.Log("Parsing begins Orderv17:" + x);

                var orderModel = JsonConvert.DeserializeObject <OrderModel>(x);

                if (orderModel != null)
                {
                    //// Construct the key //
                    var key = orderModel.OrderId + KeySeparator + orderModel.Email;

                    _logger.Log("Key " + key);

                    Document document = new Document();
                    document[OrderIdKey] = key;
                    document[ValueIdKey] = orderStr;

                    var table = Table.LoadTable(_client, OrderTableName);
                    var doc   = table.PutItemAsync(document);
                    return(doc.Result);
                }
            }
            catch (Exception ex)
            {
                _logger.Log("Eror: " + ex.Message);
                _logger.Log("Stack Trace: " + ex.StackTrace);
                throw;
            }

            return(null);
        }
Ejemplo n.º 15
0
        public async Task Process(Amazon.EC2.Model.Instance instance, ILambdaLogger logger)
        {
            if (!instance.Tags.Any(x => x.Key.Contains("Auto")))
            {
                return;
            }

            logger.Log($"Processing Update and Tag Validation of EC2 Instance {instance.InstanceId}, Name: {instance.GetTagValueOrDefault("Name")}");

            var disableAll = instance.GetTagValueOrDefault("Terminator Disable All").ToBoolOrDefault(false);

            if (!disableAll)
            {
                var targetGroups     = instance.GetTagValueOrDefault($"Target Groups") ?? instance.GetTagValueOrDefault($"TG");
                var targetGroupsList = new List <(string name, int port)>();

                if (!targetGroups.IsNullOrEmpty())
                {
                    var arrTGNames = targetGroups.Split(',');
                    var arrTGPorts = targetGroups.Split(',');

                    while (arrTGNames.Length > arrTGPorts.Length) //extrapolate by last or default 80
                    {
                        arrTGPorts = arrTGPorts.Merge(((arrTGPorts?.Length ?? 0) > 0) ? arrTGPorts[arrTGPorts.Length - 1] : "80");
                    }

                    while (arrTGPorts.Length > arrTGNames.Length) //extrapolate names by last
                    {
                        arrTGNames = arrTGNames.Merge(arrTGNames[arrTGNames.Length - 1]);
                    }

                    for (int i = 0; i < arrTGNames.Length; i++)
                    {
                        var tg = arrTGNames[i]?.Trim(" ")?.Split(':');

                        if (tg.IsNullOrEmpty() || tg.Length != 2 ||
                            tg[0].IsNullOrWhitespace() || tg[1].IsNullOrWhitespace() ||
                            !tg[1].ToIntOrDefault(-1).InClosedInterval(1, 65535))
                        {
                            logger.Log($"WARINING!!! Invalid Target Groups (TG) Name definition '{(arrTGNames[i] ?? "undefined")}' for instance {instance.InstanceId}.");
                            continue;
                        }

                        targetGroupsList.Add((tg[0].Trim(" "), tg[1].ToInt32()));
                    }
                }

                await ParallelEx.ForAsync(0, _maxPeriodsPerInstance, async i =>
                {
                    var suffix   = i == 0 ? " 1" : $" {i + 1}";
                    var disabled = instance.GetTagValueOrDefault($"Terminator Disable{suffix}").ToBoolOrDefault(false);

                    if (disabled)
                    {
                        return;
                    }

                    var cOn   = instance.GetTagValueOrDefault($"Auto On{suffix}");
                    var cOff  = instance.GetTagValueOrDefault($"Auto Off{suffix}");
                    var cKill = instance.GetTagValueOrDefault($"Auto Kill{suffix}");
                    var cTargetGroupRegister   = instance.GetTagValueOrDefault($"Auto TGR{suffix}");
                    var cTargetGroupDeregister = instance.GetTagValueOrDefault($"Auto TGD{suffix}");

                    if (cOn.IsNullOrEmpty() &&
                        cOff.IsNullOrEmpty() &&
                        cKill.IsNullOrEmpty() &&
                        cTargetGroupRegister.IsNullOrEmpty() &&
                        cTargetGroupDeregister.IsNullOrEmpty())
                    {
                        return; //skip no instance control tagg's were found
                    }
                    var ex = await ProcessTags(instance,
                                               cOn: cOn,
                                               cOff: cOff,
                                               cKill: cKill,
                                               cTargetGroupRegister: cTargetGroupRegister,
                                               cTargetGroupDeregister: cTargetGroupDeregister,
                                               targetGroups: targetGroupsList,
                                               logger: logger).CatchExceptionAsync();
                    if (ex != null)
                    {
                        logger.Log($"Failed Update or Tag Validation of EC2 Instance {instance.InstanceId} ({instance.GetTagValueOrDefault("Name")}), Auto On: '{cOn}', Auto Off: '{cOff}', Auto Kill: '{cKill}', Exception: {ex.JsonSerializeAsPrettyException()}");
                    }
                });
            }

            logger.Log($"Finished Processing and Tag Validation of EC2 Instance {instance.InstanceId} ({instance.GetTagValueOrDefault("Name")}), DisableAll: {disableAll}");
        }
Ejemplo n.º 16
0
        public async Task FunctionHandler(ILambdaContext context)
        {
            _sw = Stopwatch.StartNew();
            Log($"Started KIRA Address Balance Oracle v0.0.1");

            _context = context;
            _logger  = _context.Logger;
            _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started");
            _verbose        = Environment.GetEnvironmentVariable("VERBOSE").ToBoolOrDefault(true);
            _maxParallelism = Environment.GetEnvironmentVariable("MAX_PARALLELISM").ToIntOrDefault(1);
            var maxDuration = Environment.GetEnvironmentVariable("MAX_DURATION").ToIntOrDefault(5 * 60 * 100); // 5 minutes default
            var configUrl   = Environment.GetEnvironmentVariable("CONFIG") ?? "https://raw.githubusercontent.com/KiraCore/cfg/main/EthereumOracle/env-mainnet";
            var maxCache    = Environment.GetEnvironmentVariable("MAX_CACHE").ToIntOrDefault(0);               // seconds

            _bucket    = Environment.GetEnvironmentVariable("BUCKET");
            _networks  = Environment.GetEnvironmentVariable("NETWORKS")?.Split(",");
            _addresses = Environment.GetEnvironmentVariable("ADDRESSES")?.Split(",");
            _apikey    = Environment.GetEnvironmentVariable("APIKEY");

            if (Environment.GetEnvironmentVariable("TEST_CONNECTION").ToBoolOrDefault(false))
            {
                Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available.");
            }

            if (!configUrl.IsNullOrEmpty() && (
                    _addresses.IsNullOrEmpty() ||
                    _networks.IsNullOrEmpty() ||
                    _bucket.IsNullOrEmpty() ||
                    _apikey.IsNullOrEmpty()))
            {
                var cfg = await HttpHelper.GET <Dictionary <string, string> >(configUrl, System.Net.HttpStatusCode.OK, timeoutSeconds : 10);

                _addresses = _addresses.IsNullOrEmpty() ? cfg.GetValueOrDefault("ADDRESSES", "")?.Split(",") : _addresses;
                _networks  = _networks.IsNullOrEmpty() ? cfg.GetValueOrDefault("NETWORKS", "")?.Split(",") : _networks;
                _bucket    = _bucket.IsNullOrEmpty() ? cfg.GetValueOrDefault("BUCKET", "") : _bucket;
                _apikey    = _apikey.IsNullOrEmpty() ? cfg.GetValueOrDefault("APIKEY", "") : _apikey;

                if (cfg.GetValueOrDefault("STOP", "false").ToBoolOrDefault(false))
                {
                    _logger.Log($"WARNING: Lambda will be halted, a STOP property found in the configuration file.");
                    return;
                }
            }

            while (_sw.ElapsedMilliseconds < maxDuration)
            {
#if (TEST)
                if (_addresses.IsNullOrEmpty())                                                                                      // test networks
                {
                    _addresses = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE".Split(","); // test adresses
                }
                if (_networks.IsNullOrEmpty())                                                                                       // test networks
                {
                    _networks = "mainnet,kovan".Split(",");
                }

                if (_bucket.IsNullOrEmpty())
                {
                    _bucket = "oracle-local.kiracore.com"; // test bucket
                }
                if (_apikey.IsNullOrEmpty())
                {
                    _apikey = "MAVTB6FJWAAMJZIR825QFZ3QSR27Q3349F"; // test key
                }
                _store = _store ?? new ETHOracleStore(_bucket);
                await CacheBalances(_networks, _addresses, _apikey);
#elif (PUBLISH)
                try
                {
                    _store = _store ?? new ETHOracleStore(_bucket);
                    await CacheBalances(_networks, _addresses, _apikey, maxCache);
                }
                catch (Exception ex)
                {
                    _logger.Log($"ERROR: => Message: '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
                }
                finally
                {
                    _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {_sw.ElapsedMilliseconds} [ms]");
                }
#endif
            }
        }
Ejemplo n.º 17
0
        public async Task FunctionHandler(ILambdaContext context)
        {
            _sw      = Stopwatch.StartNew();
            _context = context;
            _logger  = _context.Logger;
            _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started");
            _verbose                = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true);
            _masterChatId           = new ChatId(Environment.GetEnvironmentVariable("MASTER_CHAT_ID").ToLongOrDefault(-1001261081309));
            _maxParallelism         = Environment.GetEnvironmentVariable("MAX_PARALLELISM").ToIntOrDefault(0);
            _cosmosHubClientTimeout = Environment.GetEnvironmentVariable("HUB_CLIENT_TIMEOUT").ToIntOrDefault(7);
            _maxMessageAge          = Environment.GetEnvironmentVariable("MAX_MESSAGE_AGE").ToDoubleOrDefault(24 * 3600);
            _bucket     = Environment.GetEnvironmentVariable("BUCKET_NAME");
            _lambdaTime = Environment.GetEnvironmentVariable("LAMBDA_TIME").ToIntOrDefault((15 * 60 * 1000) - 5000);

            var secretName = Environment.GetEnvironmentVariable("SECRET_NAME") ?? "KiraFaucetBot";

            if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false))
            {
                Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available.");
            }

            var secret = JObject.Parse(await _SM.GetSecret(secretName));

#if (TEST)
            var accessToken = secret["test_token"]?.ToString();
            //_mnemonic = (secret["test_mnemonic"]?.ToString()).ToSecureString();
            _mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString();
            _bucket   = "kira-faucet-test";
#elif (PUBLISH)
            var accessToken = secret["token"]?.ToString();
            _mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString();
#endif
            _TBC = new TelegramBotClient(accessToken);
            _bot = await _TBC.GetMeAsync();

            Log($"[INFO] {_bot.FirstName} {_version} started! Bot Name: @{_bot.Username ?? "undefined"}, Bot Id: '{_bot.Id}', Master Chat: '{_masterChatId.Identifier}'");

            _TBC.OnMessage            += Tbc_OnMessage;
            _TBC.OnCallbackQuery      += _TBC_OnCallbackQuery;
            _TBC.OnInlineQuery        += _TBC_OnInlineQuery;
            _TBC.OnInlineResultChosen += _TBC_OnInlineResultChosen;
            _TBC.StartReceiving();

            try
            {
                Log($"Processing...");
                var finalize = false;
                while (true)
                {
#if (PUBLISH)
                    if (!finalize && _sw.ElapsedMilliseconds >= _lambdaTime)
                    {
                        _TBC.StopReceiving();
                        finalize = true;
                        _logger.Log($"Finalizing, elapsed {_sw.ElapsedMilliseconds} / {_lambdaTime} [ms] ...");
                    }
#endif

                    if (_messages.IsNullOrEmpty() && _callbacks.IsNullOrEmpty())
                    {
                        if (finalize)
                        {
                            _logger.Log($"Lambda was finalized gracefully within {_lambdaTime - _sw.ElapsedMilliseconds} ms.");
                            return;
                        }
                        else
                        {
                            await Task.Delay(100);

                            continue;
                        }
                    }

                    Message[] msgArr = null;
                    _ssMsgLocker.Lock(() =>
                    {
                        msgArr = _messages.ToArray().DeepCopy();
                        _messages.Clear();
                    });

                    var t0 = ParallelEx.ForEachAsync(msgArr, async msg => {
                        async Task ProcessUser(Message m)
                        {
                            var user      = m.From;
                            var replyUser = m.ReplyToMessage?.From;

                            if (user != null)
                            {
                                await UpdateUserData(user);
                            }

                            if (replyUser != null && user?.Id != replyUser.Id)
                            {
                                await UpdateUserData(replyUser);
                            }
                        }

#if (TEST)
                        await ProcessUser(msg);
#elif (PUBLISH)
                        try
                        {
                            await ProcessUser(msg);
                        }
                        catch (Exception ex)
                        {
                            _logger.Log($"[USR ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to save user status: '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
                        }
#endif
                    });

                    var t1 = ParallelEx.ForEachAsync(msgArr, async msg =>
                    {
#if (TEST)
                        await ProcessMessage(msg);
#elif (PUBLISH)
                        try
                        {
                            await ProcessMessage(msg);
                        }
                        catch (Exception ex)
                        {
                            _logger.Log($"[MSG ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to process message ({msg?.MessageId}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
                            await _TBC.SendTextMessageAsync(chatId: msg.Chat,
                                                            $"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.",
                                                            replyToMessageId: msg.MessageId,
                                                            parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown);
                        }
#endif
                    }, maxDegreeOfParallelism: _maxParallelism);

                    CallbackQuery[] cbqArr = null;
                    _ssCbqLocker.Lock(() =>
                    {
                        cbqArr = _callbacks.ToArray().DeepCopy();
                        _callbacks.Clear();
                    });
                    var t2 = ParallelEx.ForEachAsync(cbqArr, async cbq =>
                    {
#if (TEST)
                        await ProcessCallbacks(cbq);
#elif (PUBLISH)
                        try
                        {
                            await ProcessCallbacks(cbq);
                        }
                        catch (Exception ex)
                        {
                            _logger.Log($"[CBQ ERROR] => Filed ('{cbq.Message?.Chat?.Id ?? 0}') to process callback ({cbq.Id}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
                            await _TBC.SendTextMessageAsync(chatId: cbq.Message.Chat,
                                                            $"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.",
                                                            parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown);
                        }
#endif
                    }, maxDegreeOfParallelism: _maxParallelism);

                    await Task.WhenAll(t0, t1, t2);
                }
            }
            finally
            {
                _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {_sw.ElapsedMilliseconds} [ms]");
            }
        }
 public void ReportError(TJourney journey, TimeSpan duration, Exception exception)
 {
     _errorCount++;
     _log.Log("[ERROR]" + exception);
     _journeyTime += duration;
 }