/// <summary> /// This method fails or times out with at least 20% chance /// </summary> private Task CallUnpredictableExternalAPI(SQSEvent.SQSMessage message, ILambdaLogger contextLogger) { contextLogger.Log(" ** inside CallExternalService **"); //about 10% of requests should explode var seed = Guid.NewGuid().GetHashCode(); contextLogger.Log($" ** checking RND seed {seed} **"); var rnd = new Random(seed); int failureChance = rnd.Next(1, 11); if (failureChance == 5) { contextLogger.Log($" ** about to throw exception, failureChance = {failureChance} **"); throw new ApplicationException(); } //about 10% of requests should time out failureChance = rnd.Next(1, 11); if (failureChance == 10) { contextLogger.Log($" ** about to freeze, failureChance = {failureChance} **"); Thread.Sleep(TimeOutSeconds * 1000); return(Task.Run(() => {})); } //this is happy path return(this.ResendMessageToAnotherQueue(OutputQueueUrl, message, contextLogger)); }
public async Task FunctionHandler(ILambdaContext context) { _context = context; _logger = _context.Logger; _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started"); _verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true); var maxTime = 60 * 1000; var rateLimit = 20 * 1000; var sw = Stopwatch.StartNew(); long remaining = 0; do { _logger.Log($"New Execution Started, currently elpased: {sw.ElapsedMilliseconds}"); var elapsed = await Execute(); var rateAwait = rateLimit - elapsed; remaining = maxTime - (sw.ElapsedMilliseconds + Math.Max(rateLimit, 2 * elapsed) + 1000); if (rateAwait > 0 && remaining > 0) { Log($"Rate Limit Awaiting {rateAwait} [ms] remaining out of {rateLimit} [ms]. Total elapsed: {sw.ElapsedMilliseconds} [ms]"); await Task.Delay((int)rateAwait); } await Task.Delay(1000); } while (remaining > 0); }
public async Task FunctionHandler(ILambdaContext context) { var sw = Stopwatch.StartNew(); _context = context; _logger = _context.Logger; _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started"); _verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true); var githubToken = Environment.GetEnvironmentVariable("github_token"); if (githubToken.IsNullOrEmpty()) { throw new Exception("Environment Variable 'github-token' was not defined!"); } if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false)) { Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available."); } string accessToken; if (githubToken.IsHex()) { Log($"Property 'githubToken' was determined to be a hexadecimal and will be used as accessToken"); accessToken = githubToken; } else { Log($"Fetching github access token '{githubToken ?? "undefined"}' from secrets manager."); accessToken = (await _SM.GetSecret(githubToken)).JsonDeserialize <AmazonSecretsToken>()?.token; } var rateLimit = Environment.GetEnvironmentVariable("github_rate_limit").ToIntOrDefault(1000); _GIT = new GitHubHelper(new GITWrapper.GitHub.Models.GitHubRepoConfig(accessToken: accessToken, maxRatePerHour: rateLimit) { user = Environment.GetEnvironmentVariable("github_user"), branch = Environment.GetEnvironmentVariable("github_branch"), repository = Environment.GetEnvironmentVariable("github_repository"), userAgent = Environment.GetEnvironmentVariable("user_agent") ?? "Asmodat Launcher Toolkit", }); try { Log($"Processing..."); await Processing(); } finally { _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {sw.ElapsedMilliseconds} [ms]"); await Task.Delay(2500); } }
private async Task ChangeVisibilityTimeOutAsyncTask(SQSEvent.SQSMessage message, ILambdaLogger contextLogger, int timeoutSeconds) { contextLogger.Log($" *** about to change visibility timeout to {timeoutSeconds} seconds***"); var batRequest = new ChangeMessageVisibilityRequest { ReceiptHandle = message.ReceiptHandle, VisibilityTimeout = timeoutSeconds, QueueUrl = InputQueueUrl }; var response = await this.SqSClient.ChangeMessageVisibilityAsync(batRequest); contextLogger.Log($" *** visibility timeout result status code is {response.HttpStatusCode} ***"); }
private async Task <CloudTrailRecords> ExtractCloudTrailRecordsAsync(ILambdaLogger logger, byte[] input) { var appearsGzipped = ResponseAppearsGzipped(input); logger.LogLine($"Input appears to be gzipped: {appearsGzipped}"); if (appearsGzipped) { using (var contents = new MemoryStream()) using (var gz = new GZipStream(new MemoryStream(input), CompressionMode.Decompress)) { await gz.CopyToAsync(contents); input = contents.ToArray(); } } var serializedRecords = Encoding.UTF8.GetString(input); logger.Log(serializedRecords); return(JsonConvert.DeserializeObject <CloudTrailRecords>(serializedRecords)); bool ResponseAppearsGzipped(byte[] bytes) { var header = new byte[GZipHeaderBytes.Length]; Array.Copy(bytes, header, header.Length); return(header.SequenceEqual(GZipHeaderBytes)); } }
public override void Write(string value) { _logger.Log(value); if (this._writeToInnerWriter) { this._innerWriter?.Write(value); } }
private void Log(string msg) { if (msg.IsNullOrEmpty() || !_verbose) { return; } _logger.Log(msg); }
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter) { string message = string.Empty; if (formatter != null) { message = formatter(state, exception); } lambdaLogger.Log(message); }
public void Send(DataPointUploadMessage msg) { try { var request = _requestor.GetRequestor(); using (var rs = request.GetWriteStream()) { Serializer.Serialize(rs, msg); // flush the message before disposing rs.Flush(); } try { using (request.Send()) { } } catch (SecurityException) { log.Log("API token for sending metrics to SignalFx is invalid"); } } catch (Exception ex) { if (ex is WebException) { var webex = ex as WebException; using (var exresp = webex.Response) { if (exresp != null) { var stream2 = exresp.GetResponseStream(); var reader2 = new StreamReader(stream2); var errorStr = reader2.ReadToEnd(); log.Log(errorStr); } } } } }
private void InternalLog(LogMessage logMessage) { var options = new JsonSerializerOptions { WriteIndented = true, IgnoreNullValues = true }; options.Converters.Add(new JsonStringEnumConverter()); options.Converters.Add(new LogMessageJsonConverter()); var jsonString = JsonSerializer.Serialize(logMessage, options); lambdaLogger.Log(jsonString); }
private SkillResponse HandleNextEventRequest(IntentRequest intentRequest, ILambdaLogger logger) { string message; if (intentRequest.Intent.Slots.TryGetValue("calendar_name", out var calendarNameSlot) && TryParseCalendarName(calendarNameSlot, out CalendarType? calendarType, logger) && calendarType != CalendarType.None) { Calendar calendar = CalendarService.GetCalendar(calendarType.Value) .ConfigureAwait(false).GetAwaiter().GetResult(); CalendarEntry nextEvent = CalendarService.GetFutureCalendarEntries(calendar) .FirstOrDefault(); if (nextEvent != null) { string ssml = CreateSsmlForEvent(calendar, nextEvent); logger.Log($"Created response SSML: {ssml}"); return(ResponseBuilder.Tell(new SsmlOutputSpeech { Ssml = ssml })); } else { message = $"I'm sorry. I couldn't find any future calendar entries in the {calendar.Name} calendar."; } } else { message = "I'm sorry. I wasn't able to understand that calendar name."; } var response = ResponseBuilder.Tell(new PlainTextOutputSpeech { Text = message }); response.Response.ShouldEndSession = true; return(response); }
private bool TryParseCalendarName(Slot calendarNameSlot, out CalendarType?calendarType, ILambdaLogger logger) { string calendarTypeText = calendarNameSlot.Value; var firstResolution = calendarNameSlot.Resolution?.Authorities?.FirstOrDefault(); var firstResolutionStatusCode = firstResolution?.Status?.Code; var firstResolutionValue = firstResolution?.Values?.FirstOrDefault()?.Value?.Name; if (firstResolutionStatusCode == "ER_SUCCESS_MATCH" && !String.IsNullOrEmpty(firstResolutionValue)) { calendarTypeText = firstResolutionValue; } logger.Log($"Received request for calendar: {calendarNameSlot.Value} - Resolution code: {firstResolutionStatusCode} - Resolution value: {firstResolutionValue} - Resolved to: {calendarTypeText}"); if (Enum.TryParse(typeof(CalendarType), calendarTypeText, out var tempCalendarType)) { calendarType = (CalendarType)tempCalendarType; return(true); } calendarType = null; return(false); }
private async Task <SendMessageResponse> ResendMessageToAnotherQueue(string queue, SQSEvent.SQSMessage message, ILambdaLogger logger, int delaySeconds = 0) { logger.Log($" ** sending message {message.Body} to queue {queue} ** "); var writeMessageRequest = new SendMessageRequest(queue, message.Body); int retryCounter = 0; if (message.MessageAttributes.ContainsKey(RetryCount)) { retryCounter = Convert.ToInt32(message.MessageAttributes[RetryCount].StringValue); retryCounter++; } writeMessageRequest.MessageAttributes = new Dictionary <string, MessageAttributeValue>(); writeMessageRequest.MessageAttributes.Add(RetryCount, new MessageAttributeValue() { DataType = "String", StringValue = (retryCounter).ToString() }); //Normalize distribution of incoming messages in time by function x2 writeMessageRequest.DelaySeconds = retryCounter * retryCounter * delaySeconds; return(await this.SqSClient.SendMessageAsync(writeMessageRequest)); }
public async Task <Document> CreateOrder(string orderStr) { try { _logger.Log($"Raw data : {orderStr}"); var config = new AmazonDynamoDBConfig(); config.RegionEndpoint = RegionEndpoint.APSoutheast2; _client = new AmazonDynamoDBClient(config); var finalText = orderStr.Trim().Replace(@"\", ""); _logger.Log($"After replace text: {finalText}"); var x = finalText.Substring(1, finalText.Length - 2); _logger.Log("Parsing begins Orderv17:" + x); var orderModel = JsonConvert.DeserializeObject <OrderModel>(x); if (orderModel != null) { //// Construct the key // var key = orderModel.OrderId + KeySeparator + orderModel.Email; _logger.Log("Key " + key); Document document = new Document(); document[OrderIdKey] = key; document[ValueIdKey] = orderStr; var table = Table.LoadTable(_client, OrderTableName); var doc = table.PutItemAsync(document); return(doc.Result); } } catch (Exception ex) { _logger.Log("Eror: " + ex.Message); _logger.Log("Stack Trace: " + ex.StackTrace); throw; } return(null); }
public async Task Process(Amazon.EC2.Model.Instance instance, ILambdaLogger logger) { if (!instance.Tags.Any(x => x.Key.Contains("Auto"))) { return; } logger.Log($"Processing Update and Tag Validation of EC2 Instance {instance.InstanceId}, Name: {instance.GetTagValueOrDefault("Name")}"); var disableAll = instance.GetTagValueOrDefault("Terminator Disable All").ToBoolOrDefault(false); if (!disableAll) { var targetGroups = instance.GetTagValueOrDefault($"Target Groups") ?? instance.GetTagValueOrDefault($"TG"); var targetGroupsList = new List <(string name, int port)>(); if (!targetGroups.IsNullOrEmpty()) { var arrTGNames = targetGroups.Split(','); var arrTGPorts = targetGroups.Split(','); while (arrTGNames.Length > arrTGPorts.Length) //extrapolate by last or default 80 { arrTGPorts = arrTGPorts.Merge(((arrTGPorts?.Length ?? 0) > 0) ? arrTGPorts[arrTGPorts.Length - 1] : "80"); } while (arrTGPorts.Length > arrTGNames.Length) //extrapolate names by last { arrTGNames = arrTGNames.Merge(arrTGNames[arrTGNames.Length - 1]); } for (int i = 0; i < arrTGNames.Length; i++) { var tg = arrTGNames[i]?.Trim(" ")?.Split(':'); if (tg.IsNullOrEmpty() || tg.Length != 2 || tg[0].IsNullOrWhitespace() || tg[1].IsNullOrWhitespace() || !tg[1].ToIntOrDefault(-1).InClosedInterval(1, 65535)) { logger.Log($"WARINING!!! Invalid Target Groups (TG) Name definition '{(arrTGNames[i] ?? "undefined")}' for instance {instance.InstanceId}."); continue; } targetGroupsList.Add((tg[0].Trim(" "), tg[1].ToInt32())); } } await ParallelEx.ForAsync(0, _maxPeriodsPerInstance, async i => { var suffix = i == 0 ? " 1" : $" {i + 1}"; var disabled = instance.GetTagValueOrDefault($"Terminator Disable{suffix}").ToBoolOrDefault(false); if (disabled) { return; } var cOn = instance.GetTagValueOrDefault($"Auto On{suffix}"); var cOff = instance.GetTagValueOrDefault($"Auto Off{suffix}"); var cKill = instance.GetTagValueOrDefault($"Auto Kill{suffix}"); var cTargetGroupRegister = instance.GetTagValueOrDefault($"Auto TGR{suffix}"); var cTargetGroupDeregister = instance.GetTagValueOrDefault($"Auto TGD{suffix}"); if (cOn.IsNullOrEmpty() && cOff.IsNullOrEmpty() && cKill.IsNullOrEmpty() && cTargetGroupRegister.IsNullOrEmpty() && cTargetGroupDeregister.IsNullOrEmpty()) { return; //skip no instance control tagg's were found } var ex = await ProcessTags(instance, cOn: cOn, cOff: cOff, cKill: cKill, cTargetGroupRegister: cTargetGroupRegister, cTargetGroupDeregister: cTargetGroupDeregister, targetGroups: targetGroupsList, logger: logger).CatchExceptionAsync(); if (ex != null) { logger.Log($"Failed Update or Tag Validation of EC2 Instance {instance.InstanceId} ({instance.GetTagValueOrDefault("Name")}), Auto On: '{cOn}', Auto Off: '{cOff}', Auto Kill: '{cKill}', Exception: {ex.JsonSerializeAsPrettyException()}"); } }); } logger.Log($"Finished Processing and Tag Validation of EC2 Instance {instance.InstanceId} ({instance.GetTagValueOrDefault("Name")}), DisableAll: {disableAll}"); }
public async Task FunctionHandler(ILambdaContext context) { _sw = Stopwatch.StartNew(); Log($"Started KIRA Address Balance Oracle v0.0.1"); _context = context; _logger = _context.Logger; _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started"); _verbose = Environment.GetEnvironmentVariable("VERBOSE").ToBoolOrDefault(true); _maxParallelism = Environment.GetEnvironmentVariable("MAX_PARALLELISM").ToIntOrDefault(1); var maxDuration = Environment.GetEnvironmentVariable("MAX_DURATION").ToIntOrDefault(5 * 60 * 100); // 5 minutes default var configUrl = Environment.GetEnvironmentVariable("CONFIG") ?? "https://raw.githubusercontent.com/KiraCore/cfg/main/EthereumOracle/env-mainnet"; var maxCache = Environment.GetEnvironmentVariable("MAX_CACHE").ToIntOrDefault(0); // seconds _bucket = Environment.GetEnvironmentVariable("BUCKET"); _networks = Environment.GetEnvironmentVariable("NETWORKS")?.Split(","); _addresses = Environment.GetEnvironmentVariable("ADDRESSES")?.Split(","); _apikey = Environment.GetEnvironmentVariable("APIKEY"); if (Environment.GetEnvironmentVariable("TEST_CONNECTION").ToBoolOrDefault(false)) { Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available."); } if (!configUrl.IsNullOrEmpty() && ( _addresses.IsNullOrEmpty() || _networks.IsNullOrEmpty() || _bucket.IsNullOrEmpty() || _apikey.IsNullOrEmpty())) { var cfg = await HttpHelper.GET <Dictionary <string, string> >(configUrl, System.Net.HttpStatusCode.OK, timeoutSeconds : 10); _addresses = _addresses.IsNullOrEmpty() ? cfg.GetValueOrDefault("ADDRESSES", "")?.Split(",") : _addresses; _networks = _networks.IsNullOrEmpty() ? cfg.GetValueOrDefault("NETWORKS", "")?.Split(",") : _networks; _bucket = _bucket.IsNullOrEmpty() ? cfg.GetValueOrDefault("BUCKET", "") : _bucket; _apikey = _apikey.IsNullOrEmpty() ? cfg.GetValueOrDefault("APIKEY", "") : _apikey; if (cfg.GetValueOrDefault("STOP", "false").ToBoolOrDefault(false)) { _logger.Log($"WARNING: Lambda will be halted, a STOP property found in the configuration file."); return; } } while (_sw.ElapsedMilliseconds < maxDuration) { #if (TEST) if (_addresses.IsNullOrEmpty()) // test networks { _addresses = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE".Split(","); // test adresses } if (_networks.IsNullOrEmpty()) // test networks { _networks = "mainnet,kovan".Split(","); } if (_bucket.IsNullOrEmpty()) { _bucket = "oracle-local.kiracore.com"; // test bucket } if (_apikey.IsNullOrEmpty()) { _apikey = "MAVTB6FJWAAMJZIR825QFZ3QSR27Q3349F"; // test key } _store = _store ?? new ETHOracleStore(_bucket); await CacheBalances(_networks, _addresses, _apikey); #elif (PUBLISH) try { _store = _store ?? new ETHOracleStore(_bucket); await CacheBalances(_networks, _addresses, _apikey, maxCache); } catch (Exception ex) { _logger.Log($"ERROR: => Message: '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'"); } finally { _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {_sw.ElapsedMilliseconds} [ms]"); } #endif } }
public async Task FunctionHandler(ILambdaContext context) { _sw = Stopwatch.StartNew(); _context = context; _logger = _context.Logger; _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started"); _verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true); _masterChatId = new ChatId(Environment.GetEnvironmentVariable("MASTER_CHAT_ID").ToLongOrDefault(-1001261081309)); _maxParallelism = Environment.GetEnvironmentVariable("MAX_PARALLELISM").ToIntOrDefault(0); _cosmosHubClientTimeout = Environment.GetEnvironmentVariable("HUB_CLIENT_TIMEOUT").ToIntOrDefault(7); _maxMessageAge = Environment.GetEnvironmentVariable("MAX_MESSAGE_AGE").ToDoubleOrDefault(24 * 3600); _bucket = Environment.GetEnvironmentVariable("BUCKET_NAME"); _lambdaTime = Environment.GetEnvironmentVariable("LAMBDA_TIME").ToIntOrDefault((15 * 60 * 1000) - 5000); var secretName = Environment.GetEnvironmentVariable("SECRET_NAME") ?? "KiraFaucetBot"; if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false)) { Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available."); } var secret = JObject.Parse(await _SM.GetSecret(secretName)); #if (TEST) var accessToken = secret["test_token"]?.ToString(); //_mnemonic = (secret["test_mnemonic"]?.ToString()).ToSecureString(); _mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString(); _bucket = "kira-faucet-test"; #elif (PUBLISH) var accessToken = secret["token"]?.ToString(); _mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString(); #endif _TBC = new TelegramBotClient(accessToken); _bot = await _TBC.GetMeAsync(); Log($"[INFO] {_bot.FirstName} {_version} started! Bot Name: @{_bot.Username ?? "undefined"}, Bot Id: '{_bot.Id}', Master Chat: '{_masterChatId.Identifier}'"); _TBC.OnMessage += Tbc_OnMessage; _TBC.OnCallbackQuery += _TBC_OnCallbackQuery; _TBC.OnInlineQuery += _TBC_OnInlineQuery; _TBC.OnInlineResultChosen += _TBC_OnInlineResultChosen; _TBC.StartReceiving(); try { Log($"Processing..."); var finalize = false; while (true) { #if (PUBLISH) if (!finalize && _sw.ElapsedMilliseconds >= _lambdaTime) { _TBC.StopReceiving(); finalize = true; _logger.Log($"Finalizing, elapsed {_sw.ElapsedMilliseconds} / {_lambdaTime} [ms] ..."); } #endif if (_messages.IsNullOrEmpty() && _callbacks.IsNullOrEmpty()) { if (finalize) { _logger.Log($"Lambda was finalized gracefully within {_lambdaTime - _sw.ElapsedMilliseconds} ms."); return; } else { await Task.Delay(100); continue; } } Message[] msgArr = null; _ssMsgLocker.Lock(() => { msgArr = _messages.ToArray().DeepCopy(); _messages.Clear(); }); var t0 = ParallelEx.ForEachAsync(msgArr, async msg => { async Task ProcessUser(Message m) { var user = m.From; var replyUser = m.ReplyToMessage?.From; if (user != null) { await UpdateUserData(user); } if (replyUser != null && user?.Id != replyUser.Id) { await UpdateUserData(replyUser); } } #if (TEST) await ProcessUser(msg); #elif (PUBLISH) try { await ProcessUser(msg); } catch (Exception ex) { _logger.Log($"[USR ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to save user status: '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'"); } #endif }); var t1 = ParallelEx.ForEachAsync(msgArr, async msg => { #if (TEST) await ProcessMessage(msg); #elif (PUBLISH) try { await ProcessMessage(msg); } catch (Exception ex) { _logger.Log($"[MSG ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to process message ({msg?.MessageId}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'"); await _TBC.SendTextMessageAsync(chatId: msg.Chat, $"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.", replyToMessageId: msg.MessageId, parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown); } #endif }, maxDegreeOfParallelism: _maxParallelism); CallbackQuery[] cbqArr = null; _ssCbqLocker.Lock(() => { cbqArr = _callbacks.ToArray().DeepCopy(); _callbacks.Clear(); }); var t2 = ParallelEx.ForEachAsync(cbqArr, async cbq => { #if (TEST) await ProcessCallbacks(cbq); #elif (PUBLISH) try { await ProcessCallbacks(cbq); } catch (Exception ex) { _logger.Log($"[CBQ ERROR] => Filed ('{cbq.Message?.Chat?.Id ?? 0}') to process callback ({cbq.Id}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'"); await _TBC.SendTextMessageAsync(chatId: cbq.Message.Chat, $"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.", parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown); } #endif }, maxDegreeOfParallelism: _maxParallelism); await Task.WhenAll(t0, t1, t2); } } finally { _logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {_sw.ElapsedMilliseconds} [ms]"); } }
public void ReportError(TJourney journey, TimeSpan duration, Exception exception) { _errorCount++; _log.Log("[ERROR]" + exception); _journeyTime += duration; }