/// <summary> /// コスト超過しているか確認する /// </summary> /// <param name="awsAccessKey">AWSアクセスキー</param> /// <param name="awsSecretKey">AWSシークレットキー</param> /// <param name="limit">超過とみなす閾値</param> private static string getLimitoverCost(string awsAccessKey, string awsSecretKey, decimal limit) { AmazonCostExplorerClient client = new AmazonCostExplorerClient(awsAccessKey, awsSecretKey, RegionEndpoint.USEast1); GetCostAndUsageRequest req = new GetCostAndUsageRequest(); req.TimePeriod = new DateInterval(); req.TimePeriod.Start = DateTime.Now.AddDays(-3).ToString("yyyy-MM-dd"); // 3日前~本日までチェック req.TimePeriod.End = DateTime.Now.ToString("yyyy-MM-dd"); req.Granularity = Granularity.DAILY; req.Metrics = new List <string>() { "AMORTIZED_COST" }; StringBuilder sb = new StringBuilder(); // 超過情報 GetCostAndUsageResponse cost = client.GetCostAndUsage(req); for (int i = 0; i < cost.ResultsByTime.Count; i++) { string start = cost.ResultsByTime[i].TimePeriod.Start; string end = cost.ResultsByTime[i].TimePeriod.End; foreach (string key in cost.ResultsByTime[i].Total.Keys) { decimal amount = decimal.Parse(cost.ResultsByTime[i].Total[key].Amount); string unit = cost.ResultsByTime[i].Total[key].Unit; if (amount > limit) // 閾値を超えた { sb.Append(start + "~" + end + " Amount=" + amount + " Unit=" + unit + Environment.NewLine); } } } return(sb.ToString()); }
/// <summary> /// Unmarshaller the response from the service to the response class. /// </summary> /// <param name="context"></param> /// <returns></returns> public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { GetCostAndUsageResponse response = new GetCostAndUsageResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("GroupDefinitions", targetDepth)) { var unmarshaller = new ListUnmarshaller <GroupDefinition, GroupDefinitionUnmarshaller>(GroupDefinitionUnmarshaller.Instance); response.GroupDefinitions = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("NextPageToken", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.NextPageToken = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ResultsByTime", targetDepth)) { var unmarshaller = new ListUnmarshaller <ResultByTime, ResultByTimeUnmarshaller>(ResultByTimeUnmarshaller.Instance); response.ResultsByTime = unmarshaller.Unmarshall(context); continue; } } return(response); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("application started.."); try { string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); log.LogInformation("requestBody : " + requestBody); GetCostAndUsageRequest costAndUsageRequest = JsonConvert.DeserializeObject <GetCostAndUsageRequest>(requestBody); //GetCostAndUsageRequest costAndUsageRequest = BuildAWSCostAndUsageRequest(data); log.LogInformation("Build JSON Completed and AWS call initiated.."); GetCostAndUsageResponse costAndUsageResponse = CallAWSCostAndUsageAPI(costAndUsageRequest); log.LogInformation("AWS call Completed.."); CanonicalResponse canonicalresponse = CreateCanonicalData(costAndUsageResponse); log.LogInformation("Canonical response completed.."); // Insert JSON Response into Cosmos DB ProcessCosmosDB cosmosDB = new ProcessCosmosDB(); log.LogInformation("DB initiated.."); await cosmosDB.GetStartedDemoAsync(canonicalresponse); log.LogInformation("processed response completed.."); return(new OkObjectResult(costAndUsageResponse)); } catch (Exception ex) { log.LogInformation("Error occured in AWS Cost and Usage " + ex.Message.ToString()); return(new BadRequestObjectResult(ex.Message.ToString())); } }
private static CanonicalCostandUsage createCanonicalData(GetCostAndUsageResponse awsResponse) { var config = new MapperConfiguration(cfg => { cfg.CreateMap <GetCostAndUsageResponse, CanonicalCostandUsage>(); }); IMapper mapper = config.CreateMapper(); CanonicalCostandUsage genericUsage = mapper.Map <GetCostAndUsageResponse, CanonicalCostandUsage>(awsResponse); return(genericUsage); }
private static GetCostAndUsageResponse buildResponse() { GetCostAndUsageResponse costAndUsageResponse = new GetCostAndUsageResponse(); string jsonString = File.ReadAllText(@"..\TestData\costandusageresponse.json"); costAndUsageResponse = JsonSerializer.Deserialize <GetCostAndUsageResponse>(jsonString); return(costAndUsageResponse); }
//private static GetCostAndUsageRequest BuildAWSCostAndUsageRequest(object data) //{ // GetCostAndUsageRequest costAndUsageRequest = new GetCostAndUsageRequest(); // //string jsonString = File.ReadAllText(@"costandusageresquest.json"); // costAndUsageRequest = JsonConvert.DeserializeObject<GetCostAndUsageRequest>(data.ToString()); // return costAndUsageRequest; //} private static GetCostAndUsageResponse CallAWSCostAndUsageAPI(GetCostAndUsageRequest costAndUsageRequest) { var client = new AmazonCostExplorerClient( awsAccessKeyId: Environment.GetEnvironmentVariable("awsAccessKeyId"), awsSecretAccessKey: Environment.GetEnvironmentVariable("awsSecretAccessKey"), Amazon.RegionEndpoint.USEast1); GetCostAndUsageResponse costAndUsageResponse = client.GetCostAndUsageAsync(costAndUsageRequest).Result; return(costAndUsageResponse); }
private static CanonicalResponse CreateCanonicalData(GetCostAndUsageResponse awsResponse) { var config = new MapperConfiguration(cfg => { cfg.CreateMap <Amazon.CostExplorer.Model.ResultByTime, ResultByTime>(); cfg.CreateMap <Amazon.CostExplorer.Model.DateInterval, DateInterval>(); cfg.CreateMap <Amazon.CostExplorer.Model.MetricValue, MetricValue>(); cfg.CreateMap <Amazon.CostExplorer.Model.Group, Group>(); cfg.CreateMap <Amazon.Runtime.ResponseMetadata, ResponseMetadata>(); cfg.CreateMap <GetCostAndUsageResponse, CanonicalResponse>(); }); IMapper mapper = config.CreateMapper(); CanonicalResponse genericUsage = mapper.Map <GetCostAndUsageResponse, CanonicalResponse>(awsResponse); genericUsage.id = Guid.NewGuid(); // genericUsage.id = awsResponse.ResponseMetadata.RequestId; genericUsage.Company = "Advantasure"; genericUsage.Product = "Symphony"; genericUsage.Provider = "AWS"; return(genericUsage); }
public void FunctionHandler(Stream inputStream, ILambdaContext context) { Amazon.XRay.Recorder.Handlers.AwsSdk.AWSSDKHandler.RegisterXRayForAllServices(); TextReader textReader = new StreamReader(inputStream); var strInput = textReader.ReadToEnd(); LambdaLogger.Log($"Received input {strInput}"); var message = JsonConvert.DeserializeObject <SqsEvents>(strInput); // Could have multiple records in the message from SQS if (message.Records == null || message.Records.Length <= 0) { return; } List <string> lstJobIds = new List <string>(); Parallel.ForEach(message.Records, (record) => { var processRequest = JsonConvert.DeserializeObject <UsageTypeProcessRequest>(Regex.Unescape(record.body)); LambdaLogger.Log($"Received request to process usage type {processRequest.UsageType }"); if (!lstJobIds.Contains(processRequest.JobId)) { lstJobIds.Add(processRequest.JobId); } int LookbackDays = GetTotalLookbackDays(); GetCostAndUsageResponse results = null; bool blnRetry = false; int intSleepDuration = 0; int intMaxWaitInterval = 5000; int intRetries = 0; int intMaxRetries = 10; do { try { results = costExplorerClient.Value.GetCostAndUsageAsync(new Amazon.CostExplorer.Model.GetCostAndUsageRequest { TimePeriod = new Amazon.CostExplorer.Model.DateInterval { End = DateTime.Now.ToString("yyyy-MM-dd"), // We need to add one day because yesterday is when the actual usage // analysis is important - it should be excluded from the average calculation Start = DateTime.Now.AddDays(-1 * (LookbackDays + 1)).ToString("yyyy-MM-dd") }, Filter = BuildCostExplorerFilter(processRequest.UsageType), Granularity = Amazon.CostExplorer.Granularity.DAILY, Metrics = new List <string>(new string[] { "UsageQuantity", "AmortizedCost" }) }).GetAwaiter().GetResult(); blnRetry = false; } catch (Amazon.CostExplorer.Model.LimitExceededException) // Use exponential backoff to wait until the API is ready again { blnRetry = true; intSleepDuration = Convert.ToInt32((long)Math.Pow(2, intRetries) * 100L); Thread.Sleep(Math.Min(intMaxWaitInterval, intSleepDuration)); } catch (Exception ex) { blnRetry = false; LambdaLogger.Log($"An error occurred calling the Cost Explorer API for usage type {processRequest.UsageType}: {ex.Message}"); } } while (blnRetry && (intRetries++ < intMaxRetries)); if (results == null) { IncrementControlTable(processRequest.JobId); throw new Exception("Results not retrieved - this is likely due to the Cost Explorer API timing out."); } double dblTotal = 0d; double dblYesterdayUsage = 0d; var dblAverage = 0d; bool blnTrigger = false; double dblIncreaseAmount = 0d; int intActualDays = 0; var dblMinIncreaseThreshold = GetMinIncreaseThreshold(); DateTime dtYesterday = DateTime.Now; foreach (var item in results.ResultsByTime) { var startDate = DateTime.Parse(item.TimePeriod.Start); var endDate = DateTime.Parse(item.TimePeriod.End); if (DaysOfWeek != null && !DaysOfWeek.Contains(((int)startDate.DayOfWeek + 1))) { // This day is not included in the config continue; } // Check if it is the appropriate day of week var amortizedItem = item.Total.FirstOrDefault(a => a.Key == "AmortizedCost"); // Debug.WriteLine($"{processRequest.UsageType} - {startDate.ToString("d MMM yyyy")} - {amortizedItem.Key} {amortizedItem.Value.Unit}{amortizedItem.Value.Amount}"); var dblValue = double.Parse(amortizedItem.Value.Amount); if (intActualDays == 0) { dblYesterdayUsage = dblValue; dtYesterday = startDate; } else { dblTotal += dblValue; } intActualDays++; } if (intActualDays > 1) // Must minus one to cater for the previous date { dblAverage = dblTotal / Convert.ToDouble(intActualDays - 1); if (dblYesterdayUsage > dblAverage) // The usage has increased { string strThreshold; double dblThreshold = 0.2d; // 20% increase by default if (new SSMParameterManager().TryGetValue(Constants.ChangeThresholdSSMPath, out strThreshold)) { double.TryParse(strThreshold, out dblThreshold); if (dblThreshold <= 0d) { dblThreshold = 0.2d; } } dblIncreaseAmount = dblYesterdayUsage - dblAverage; if (IsTestMode()) { // 50% chance of triggering blnTrigger = new Random(DateTime.Now.Millisecond).NextDouble() > 0.5d; } else { if (dblIncreaseAmount > dblMinIncreaseThreshold && dblIncreaseAmount > (dblThreshold * dblAverage)) { blnTrigger = true; } } } } var strDDBTableName = System.Environment.GetEnvironmentVariable("BatTable"); var ddbResult = dynamoDBClient.Value.PutItemAsync(new PutItemRequest { Item = new Dictionary <string, AttributeValue>() { { "id", new AttributeValue { S = processRequest.UsageType } }, { "Total", new AttributeValue { N = dblTotal.ToString() } }, { "Processed", new AttributeValue { BOOL = true } }, { "Triggered", new AttributeValue { BOOL = blnTrigger } }, { "AverageDaily", new AttributeValue { N = dblAverage.ToString() } }, { "PreviousDay", new AttributeValue { N = dblYesterdayUsage.ToString() } }, { "IncreaseBy", new AttributeValue { N = dblIncreaseAmount.ToString() } }, { "YesterdayDate", new AttributeValue { S = dtYesterday.ToString("yyyy-MM-dd") } } }, TableName = strDDBTableName }).GetAwaiter().GetResult(); if (ddbResult.HttpStatusCode != System.Net.HttpStatusCode.OK) { LambdaLogger.Log($"WARN: Unsuccessful insert of DynamoDB table {strDDBTableName }"); } LambdaLogger.Log($"TOTAL FOR {processRequest.UsageType} - {dblTotal.ToString("C")}"); IncrementControlTable(processRequest.JobId); }); foreach (var JobId in lstJobIds) { // Check if this is the last usage type to process, invoke the finalizer string strBatFinalizerQueueUrl = System.Environment.GetEnvironmentVariable("BatFinalizerQueueUrl"); SendMessageRequest sendMessageRequest = new SendMessageRequest { MessageBody = JobId, QueueUrl = strBatFinalizerQueueUrl }; LambdaLogger.Log($"Putting message on SQS queue for Job Id {JobId}"); var sendMessageResult = sqsClient.Value.SendMessageAsync(sendMessageRequest).GetAwaiter().GetResult(); } }
public static void GetCostsAndUsage() { Console.WriteLine(Utils.CurrentMethod); GetCostAndUsageRequest request = new GetCostAndUsageRequest(); request.TimePeriod = new DateInterval() { Start = "2019-04-01", End = "2019-05-01" }; request.Granularity = Granularity.MONTHLY; request.Metrics = new List <string>(); request.Metrics.Add(Metric.AMORTIZED_COST); request.Metrics.Add(Metric.BLENDED_COST); request.Metrics.Add(Metric.NET_AMORTIZED_COST); request.Metrics.Add(Metric.NET_UNBLENDED_COST); request.Metrics.Add(Metric.NORMALIZED_USAGE_AMOUNT); request.Metrics.Add(Metric.UNBLENDED_COST); request.Metrics.Add(Metric.USAGE_QUANTITY); request.GroupBy = new List <GroupDefinition>(); request.GroupBy.Add(new GroupDefinition { Type = GroupDefinitionType.DIMENSION, Key = Dimension.LINKED_ACCOUNT }); request.GroupBy.Add(new GroupDefinition { Type = GroupDefinitionType.TAG, Key = "cloud-environment" }); // request.GroupBy.Add(new GroupDefinition { Type = GroupDefinitionType.DIMENSION, Key = Dimension.SERVICE }); bool keepLooking = true; int callCount = 0; while (keepLooking) { Task <GetCostAndUsageResponse> t = client.GetCostAndUsageAsync(request); t.Wait(30000); GetCostAndUsageResponse response = t.Result; string json = JsonTools.Serialize(response, true); Console.WriteLine(json); String fileName = $"/Users/guy/temp/cost-and-usage-{callCount.ToString().PadLeft(4, '0')}.json"; File.WriteAllText(fileName, json); string nextPageToken = response.NextPageToken; if (String.IsNullOrWhiteSpace(nextPageToken)) { keepLooking = false; } else if (callCount > 100) { keepLooking = false; } else { request.NextPageToken = nextPageToken; } callCount++; } }
public Dictionary <DateTime, IEnumerable <CostReportDetail> > EditDetailsByResponse(GetCostAndUsageResponse response) { return(response.ResultsByTime.ToDictionary( x => DateTime.ParseExact(x.TimePeriod.Start, "yyyy-MM-dd", null), x => x.Groups.Select(x => new CostReportDetail(x.Keys.First(), decimal.Parse(x.Metrics.First().Value.Amount))))); }
public async override Task <IEnumerable <CostDto> > Handle(GetMonthlyTotalCostCommand command, CancellationToken cancellationToken = default) { var result = new List <CostDto>(); using var client = _awsClientFactory.Create <AmazonCostExplorerClient>(command.AssumeProfile); try { GetCostAndUsageResponse resp = null; do { var request = new GetCostAndUsageRequest() { Metrics = new List <string>(new[] { Amazon.CostExplorer.Metric.BLENDED_COST.Value }), TimePeriod = CreateDateIntervalForCurrentMonth(), Granularity = Granularity.MONTHLY, NextPageToken = resp?.NextPageToken }; if (!string.IsNullOrEmpty(command.AccountIdentifier)) { request.Filter = new Expression { Dimensions = new DimensionValues() { Key = Dimension.LINKED_ACCOUNT, Values = new List <string>() { command.AccountIdentifier } } }; } else { request.GroupBy = new List <GroupDefinition>(new[] { new GroupDefinition() { Type = GroupDefinitionType.DIMENSION, Key = Dimension.LINKED_ACCOUNT.Value } }); } resp = await client.GetCostAndUsageAsync(request, cancellationToken); var dto = new CostDto { DimensionValueAttributes = resp.DimensionValueAttributes?.Select(o => new DimensionValueAttributeDto { Attributes = o.Attributes, Value = o.Value }), ResultsByTime = resp.ResultsByTime?.Select(o => new ResultByTimeDto { Total = o.Total?.Select(kvp => new KeyValuePair <string, MetricValueDto>(kvp.Key, new MetricValueDto { Amount = kvp.Value.Amount, Unit = kvp.Value.Unit })), StartDate = DateTime.Parse(o.TimePeriod.Start), EndDate = DateTime.Parse(o.TimePeriod.End), Estimated = o.Estimated, Groups = o.Groups?.Select(g => new GroupDto { Keys = g.Keys?.AsEnumerable(), Metrics = g.Metrics?.Select(m => new KeyValuePair <string, MetricValueDto>(m.Key, new MetricValueDto { Amount = m.Value.Amount, Unit = m.Value.Unit })) }) }) }; result.Add(dto); }while (resp.NextPageToken != null); } catch (AmazonServiceException e) { throw new Exception(e.Message, e); } return(result.AsEnumerable()); }