/// <summary>
/// コスト超過しているか確認する
/// </summary>
/// <param name="awsAccessKey">AWSアクセスキー</param>
/// <param name="awsSecretKey">AWSシークレットキー</param>
/// <param name="limit">超過とみなす閾値</param>
private static string getLimitoverCost(string awsAccessKey, string awsSecretKey, decimal limit)
{
AmazonCostExplorerClient client = new AmazonCostExplorerClient(awsAccessKey, awsSecretKey, RegionEndpoint.USEast1);
GetCostAndUsageRequest req = new GetCostAndUsageRequest();
req.TimePeriod = new DateInterval();
req.TimePeriod.Start = DateTime.Now.AddDays(-3).ToString("yyyy-MM-dd"); // 3日前～本日までチェック
req.TimePeriod.End = DateTime.Now.ToString("yyyy-MM-dd");
req.Granularity = Granularity.DAILY;
req.Metrics = new List <string>()
{
"AMORTIZED_COST"
};
StringBuilder sb = new StringBuilder(); // 超過情報
GetCostAndUsageResponse cost = client.GetCostAndUsage(req);
for (int i = 0; i < cost.ResultsByTime.Count; i++)
{
string start = cost.ResultsByTime[i].TimePeriod.Start;
string end = cost.ResultsByTime[i].TimePeriod.End;
foreach (string key in cost.ResultsByTime[i].Total.Keys)
{
decimal amount = decimal.Parse(cost.ResultsByTime[i].Total[key].Amount);
string unit = cost.ResultsByTime[i].Total[key].Unit;
if (amount > limit) // 閾値を超えた
{
sb.Append(start + "～" + end + " Amount=" + amount + " Unit=" + unit + Environment.NewLine);
}
}
}
return(sb.ToString());
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
GetCostAndUsageResponse response = new GetCostAndUsageResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("GroupDefinitions", targetDepth))
{
var unmarshaller = new ListUnmarshaller <GroupDefinition, GroupDefinitionUnmarshaller>(GroupDefinitionUnmarshaller.Instance);
response.GroupDefinitions = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("NextPageToken", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.NextPageToken = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("ResultsByTime", targetDepth))
{
var unmarshaller = new ListUnmarshaller <ResultByTime, ResultByTimeUnmarshaller>(ResultByTimeUnmarshaller.Instance);
response.ResultsByTime = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("application started..");
try
{
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
log.LogInformation("requestBody : " + requestBody);
GetCostAndUsageRequest costAndUsageRequest = JsonConvert.DeserializeObject <GetCostAndUsageRequest>(requestBody);
//GetCostAndUsageRequest costAndUsageRequest = BuildAWSCostAndUsageRequest(data);
log.LogInformation("Build JSON Completed and AWS call initiated..");
GetCostAndUsageResponse costAndUsageResponse = CallAWSCostAndUsageAPI(costAndUsageRequest);
log.LogInformation("AWS call Completed..");
CanonicalResponse canonicalresponse = CreateCanonicalData(costAndUsageResponse);
log.LogInformation("Canonical response completed..");
// Insert JSON Response into Cosmos DB
ProcessCosmosDB cosmosDB = new ProcessCosmosDB();
log.LogInformation("DB initiated..");
await cosmosDB.GetStartedDemoAsync(canonicalresponse);
log.LogInformation("processed response completed..");
return(new OkObjectResult(costAndUsageResponse));
}
catch (Exception ex)
{
log.LogInformation("Error occured in AWS Cost and Usage " + ex.Message.ToString());
return(new BadRequestObjectResult(ex.Message.ToString()));
}
}
private static CanonicalCostandUsage createCanonicalData(GetCostAndUsageResponse awsResponse)
{
var config = new MapperConfiguration(cfg => { cfg.CreateMap <GetCostAndUsageResponse, CanonicalCostandUsage>(); });
IMapper mapper = config.CreateMapper();
CanonicalCostandUsage genericUsage = mapper.Map <GetCostAndUsageResponse, CanonicalCostandUsage>(awsResponse);
return(genericUsage);
}
private static GetCostAndUsageResponse buildResponse()
{
GetCostAndUsageResponse costAndUsageResponse = new GetCostAndUsageResponse();
string jsonString = File.ReadAllText(@"..\TestData\costandusageresponse.json");
costAndUsageResponse = JsonSerializer.Deserialize <GetCostAndUsageResponse>(jsonString);
return(costAndUsageResponse);
}
//private static GetCostAndUsageRequest BuildAWSCostAndUsageRequest(object data)
//{
// GetCostAndUsageRequest costAndUsageRequest = new GetCostAndUsageRequest();
// //string jsonString = File.ReadAllText(@"costandusageresquest.json");
// costAndUsageRequest = JsonConvert.DeserializeObject<GetCostAndUsageRequest>(data.ToString());
// return costAndUsageRequest;
//}
private static GetCostAndUsageResponse CallAWSCostAndUsageAPI(GetCostAndUsageRequest costAndUsageRequest)
{
var client = new AmazonCostExplorerClient(
awsAccessKeyId: Environment.GetEnvironmentVariable("awsAccessKeyId"),
awsSecretAccessKey: Environment.GetEnvironmentVariable("awsSecretAccessKey"),
Amazon.RegionEndpoint.USEast1);
GetCostAndUsageResponse costAndUsageResponse = client.GetCostAndUsageAsync(costAndUsageRequest).Result;
return(costAndUsageResponse);
}
private static CanonicalResponse CreateCanonicalData(GetCostAndUsageResponse awsResponse)
{
var config = new MapperConfiguration(cfg => { cfg.CreateMap <Amazon.CostExplorer.Model.ResultByTime, ResultByTime>();
cfg.CreateMap <Amazon.CostExplorer.Model.DateInterval, DateInterval>();
cfg.CreateMap <Amazon.CostExplorer.Model.MetricValue, MetricValue>();
cfg.CreateMap <Amazon.CostExplorer.Model.Group, Group>();
cfg.CreateMap <Amazon.Runtime.ResponseMetadata, ResponseMetadata>();
cfg.CreateMap <GetCostAndUsageResponse, CanonicalResponse>(); });
IMapper mapper = config.CreateMapper();
CanonicalResponse genericUsage = mapper.Map <GetCostAndUsageResponse, CanonicalResponse>(awsResponse);
genericUsage.id = Guid.NewGuid();
// genericUsage.id = awsResponse.ResponseMetadata.RequestId;
genericUsage.Company = "Advantasure";
genericUsage.Product = "Symphony";
genericUsage.Provider = "AWS";
return(genericUsage);
}
public void FunctionHandler(Stream inputStream, ILambdaContext context)
{
Amazon.XRay.Recorder.Handlers.AwsSdk.AWSSDKHandler.RegisterXRayForAllServices();
TextReader textReader = new StreamReader(inputStream);
var strInput = textReader.ReadToEnd();
LambdaLogger.Log($"Received input {strInput}");
var message = JsonConvert.DeserializeObject <SqsEvents>(strInput);
// Could have multiple records in the message from SQS
if (message.Records == null || message.Records.Length <= 0)
{
return;
}
List <string> lstJobIds = new List <string>();
Parallel.ForEach(message.Records, (record) =>
{
var processRequest = JsonConvert.DeserializeObject <UsageTypeProcessRequest>(Regex.Unescape(record.body));
LambdaLogger.Log($"Received request to process usage type {processRequest.UsageType }");
if (!lstJobIds.Contains(processRequest.JobId))
{
lstJobIds.Add(processRequest.JobId);
}
int LookbackDays = GetTotalLookbackDays();
GetCostAndUsageResponse results = null;
bool blnRetry = false;
int intSleepDuration = 0;
int intMaxWaitInterval = 5000;
int intRetries = 0;
int intMaxRetries = 10;
do
{
try
{
results = costExplorerClient.Value.GetCostAndUsageAsync(new Amazon.CostExplorer.Model.GetCostAndUsageRequest
{
TimePeriod = new Amazon.CostExplorer.Model.DateInterval
{
End = DateTime.Now.ToString("yyyy-MM-dd"),
// We need to add one day because yesterday is when the actual usage
// analysis is important - it should be excluded from the average calculation
Start = DateTime.Now.AddDays(-1 * (LookbackDays + 1)).ToString("yyyy-MM-dd")
},
Filter = BuildCostExplorerFilter(processRequest.UsageType),
Granularity = Amazon.CostExplorer.Granularity.DAILY,
Metrics = new List <string>(new string[] { "UsageQuantity", "AmortizedCost" })
}).GetAwaiter().GetResult();
blnRetry = false;
}
catch (Amazon.CostExplorer.Model.LimitExceededException) // Use exponential backoff to wait until the API is ready again
{
blnRetry = true;
intSleepDuration = Convert.ToInt32((long)Math.Pow(2, intRetries) * 100L);
Thread.Sleep(Math.Min(intMaxWaitInterval, intSleepDuration));
}
catch (Exception ex)
{
blnRetry = false;
LambdaLogger.Log($"An error occurred calling the Cost Explorer API for usage type {processRequest.UsageType}: {ex.Message}");
}
} while (blnRetry && (intRetries++ < intMaxRetries));
if (results == null)
{
IncrementControlTable(processRequest.JobId);
throw new Exception("Results not retrieved - this is likely due to the Cost Explorer API timing out.");
}
double dblTotal = 0d;
double dblYesterdayUsage = 0d;
var dblAverage = 0d;
bool blnTrigger = false;
double dblIncreaseAmount = 0d;
int intActualDays = 0;
var dblMinIncreaseThreshold = GetMinIncreaseThreshold();
DateTime dtYesterday = DateTime.Now;
foreach (var item in results.ResultsByTime)
{
var startDate = DateTime.Parse(item.TimePeriod.Start);
var endDate = DateTime.Parse(item.TimePeriod.End);
if (DaysOfWeek != null && !DaysOfWeek.Contains(((int)startDate.DayOfWeek + 1)))
{
// This day is not included in the config
continue;
}
// Check if it is the appropriate day of week
var amortizedItem = item.Total.FirstOrDefault(a => a.Key == "AmortizedCost");
// Debug.WriteLine($"{processRequest.UsageType} - {startDate.ToString("d MMM yyyy")} - {amortizedItem.Key} {amortizedItem.Value.Unit}{amortizedItem.Value.Amount}");
var dblValue = double.Parse(amortizedItem.Value.Amount);
if (intActualDays == 0)
{
dblYesterdayUsage = dblValue;
dtYesterday = startDate;
}
else
{
dblTotal += dblValue;
}
intActualDays++;
}
if (intActualDays > 1) // Must minus one to cater for the previous date
{
dblAverage = dblTotal / Convert.ToDouble(intActualDays - 1);
if (dblYesterdayUsage > dblAverage) // The usage has increased
{
string strThreshold;
double dblThreshold = 0.2d; // 20% increase by default
if (new SSMParameterManager().TryGetValue(Constants.ChangeThresholdSSMPath, out strThreshold))
{
double.TryParse(strThreshold, out dblThreshold);
if (dblThreshold <= 0d)
{
dblThreshold = 0.2d;
}
}
dblIncreaseAmount = dblYesterdayUsage - dblAverage;
if (IsTestMode())
{
// 50% chance of triggering
blnTrigger = new Random(DateTime.Now.Millisecond).NextDouble() > 0.5d;
}
else
{
if (dblIncreaseAmount > dblMinIncreaseThreshold && dblIncreaseAmount > (dblThreshold * dblAverage))
{
blnTrigger = true;
}
}
}
}
var strDDBTableName = System.Environment.GetEnvironmentVariable("BatTable");
var ddbResult = dynamoDBClient.Value.PutItemAsync(new PutItemRequest
{
Item = new Dictionary <string, AttributeValue>()
{
{ "id", new AttributeValue {
S = processRequest.UsageType
} },
{ "Total", new AttributeValue {
N = dblTotal.ToString()
} },
{ "Processed", new AttributeValue {
BOOL = true
} },
{ "Triggered", new AttributeValue {
BOOL = blnTrigger
} },
{ "AverageDaily", new AttributeValue {
N = dblAverage.ToString()
} },
{ "PreviousDay", new AttributeValue {
N = dblYesterdayUsage.ToString()
} },
{ "IncreaseBy", new AttributeValue {
N = dblIncreaseAmount.ToString()
} },
{ "YesterdayDate", new AttributeValue {
S = dtYesterday.ToString("yyyy-MM-dd")
} }
},
TableName = strDDBTableName
}).GetAwaiter().GetResult();
if (ddbResult.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
LambdaLogger.Log($"WARN: Unsuccessful insert of DynamoDB table {strDDBTableName }");
}
LambdaLogger.Log($"TOTAL FOR {processRequest.UsageType} - {dblTotal.ToString("C")}");
IncrementControlTable(processRequest.JobId);
});
foreach (var JobId in lstJobIds)
{
// Check if this is the last usage type to process, invoke the finalizer
string strBatFinalizerQueueUrl = System.Environment.GetEnvironmentVariable("BatFinalizerQueueUrl");
SendMessageRequest sendMessageRequest = new SendMessageRequest
{
MessageBody = JobId,
QueueUrl = strBatFinalizerQueueUrl
};
LambdaLogger.Log($"Putting message on SQS queue for Job Id {JobId}");
var sendMessageResult = sqsClient.Value.SendMessageAsync(sendMessageRequest).GetAwaiter().GetResult();
}
}
public static void GetCostsAndUsage()
{
Console.WriteLine(Utils.CurrentMethod);
GetCostAndUsageRequest request = new GetCostAndUsageRequest();
request.TimePeriod = new DateInterval()
{
Start = "2019-04-01", End = "2019-05-01"
};
request.Granularity = Granularity.MONTHLY;
request.Metrics = new List <string>();
request.Metrics.Add(Metric.AMORTIZED_COST);
request.Metrics.Add(Metric.BLENDED_COST);
request.Metrics.Add(Metric.NET_AMORTIZED_COST);
request.Metrics.Add(Metric.NET_UNBLENDED_COST);
request.Metrics.Add(Metric.NORMALIZED_USAGE_AMOUNT);
request.Metrics.Add(Metric.UNBLENDED_COST);
request.Metrics.Add(Metric.USAGE_QUANTITY);
request.GroupBy = new List <GroupDefinition>();
request.GroupBy.Add(new GroupDefinition {
Type = GroupDefinitionType.DIMENSION, Key = Dimension.LINKED_ACCOUNT
});
request.GroupBy.Add(new GroupDefinition {
Type = GroupDefinitionType.TAG, Key = "cloud-environment"
});
// request.GroupBy.Add(new GroupDefinition { Type = GroupDefinitionType.DIMENSION, Key = Dimension.SERVICE });
bool keepLooking = true;
int callCount = 0;
while (keepLooking)
{
Task <GetCostAndUsageResponse> t = client.GetCostAndUsageAsync(request);
t.Wait(30000);
GetCostAndUsageResponse response = t.Result;
string json = JsonTools.Serialize(response, true);
Console.WriteLine(json);
String fileName = $"/Users/guy/temp/cost-and-usage-{callCount.ToString().PadLeft(4, '0')}.json";
File.WriteAllText(fileName, json);
string nextPageToken = response.NextPageToken;
if (String.IsNullOrWhiteSpace(nextPageToken))
{
keepLooking = false;
}
else if (callCount > 100)
{
keepLooking = false;
}
else
{
request.NextPageToken = nextPageToken;
}
callCount++;
}
}
public Dictionary <DateTime, IEnumerable <CostReportDetail> > EditDetailsByResponse(GetCostAndUsageResponse response)
{
return(response.ResultsByTime.ToDictionary(
x => DateTime.ParseExact(x.TimePeriod.Start, "yyyy-MM-dd", null),
x => x.Groups.Select(x => new CostReportDetail(x.Keys.First(), decimal.Parse(x.Metrics.First().Value.Amount)))));
}
public async override Task <IEnumerable <CostDto> > Handle(GetMonthlyTotalCostCommand command, CancellationToken cancellationToken = default)
{
var result = new List <CostDto>();
using var client = _awsClientFactory.Create <AmazonCostExplorerClient>(command.AssumeProfile);
try
{
GetCostAndUsageResponse resp = null;
do
{
var request = new GetCostAndUsageRequest()
{
Metrics = new List <string>(new[] { Amazon.CostExplorer.Metric.BLENDED_COST.Value }),
TimePeriod = CreateDateIntervalForCurrentMonth(),
Granularity = Granularity.MONTHLY,
NextPageToken = resp?.NextPageToken
};
if (!string.IsNullOrEmpty(command.AccountIdentifier))
{
request.Filter = new Expression
{
Dimensions = new DimensionValues()
{
Key = Dimension.LINKED_ACCOUNT,
Values = new List <string>()
{
command.AccountIdentifier
}
}
};
}
else
{
request.GroupBy = new List <GroupDefinition>(new[]
{
new GroupDefinition()
{
Type = GroupDefinitionType.DIMENSION,
Key = Dimension.LINKED_ACCOUNT.Value
}
});
}
resp = await client.GetCostAndUsageAsync(request, cancellationToken);
var dto = new CostDto
{
DimensionValueAttributes = resp.DimensionValueAttributes?.Select(o => new DimensionValueAttributeDto
{
Attributes = o.Attributes,
Value = o.Value
}),
ResultsByTime = resp.ResultsByTime?.Select(o => new ResultByTimeDto
{
Total = o.Total?.Select(kvp => new KeyValuePair <string, MetricValueDto>(kvp.Key, new MetricValueDto
{
Amount = kvp.Value.Amount,
Unit = kvp.Value.Unit
})),
StartDate = DateTime.Parse(o.TimePeriod.Start),
EndDate = DateTime.Parse(o.TimePeriod.End),
Estimated = o.Estimated,
Groups = o.Groups?.Select(g => new GroupDto
{
Keys = g.Keys?.AsEnumerable(),
Metrics = g.Metrics?.Select(m => new KeyValuePair <string, MetricValueDto>(m.Key, new MetricValueDto
{
Amount = m.Value.Amount,
Unit = m.Value.Unit
}))
})
})
};
result.Add(dto);
}while (resp.NextPageToken != null);
}
catch (AmazonServiceException e)
{
throw new Exception(e.Message, e);
}
return(result.AsEnumerable());
}
