public static void Main(string[] args)
{
// Get the last months billing data
DateTime startDateTime = DateTime.Now.Date.AddMonths(-1);
DateTime endDateTime = DateTime.Now.Date;
// Get AAD token
string token = GetOAuthTokenFromAAD(clientId, clientSecret, tennantId, resource, authority).Result;
// Get rate card
RateCardPayload rateCardPayload = GetRateCard(token, billingService, subscriptionId).Result;
// Get usage
UsagePayload usagePayload = GetUsage(token, billingService, subscriptionId, startDateTime, endDateTime).Result;
// Create aggregates by resource group
List <ResourceGroupAggregate> resourceGroupAggregates = CreateResourceGroupAggregates(usagePayload, rateCardPayload);
// Write headers to console
Console.WriteLine("Resource Group\tDate\tCost");
// Write daily output to console
foreach (ResourceGroupAggregate resourceGroupAggregate in resourceGroupAggregates)
{
foreach (DateTime date in resourceGroupAggregate.ResourceItems.SelectMany(x => x.ResourceDateItems.Select(y => y.UsageStartTime)).Distinct().ToList())
{
Console.WriteLine(resourceGroupAggregate.ResourceGroupName + "\t" + date.ToString("dd/MM/yyyy") + "\t" + resourceGroupAggregate.ResourceItems.SelectMany(x => x.ResourceDateItems.Where(y => y.UsageStartTime == date)).Sum(z => z.Cost).ToString("F"));
}
}
Console.ReadLine();
}
private static double Calculate(IEnumerable <UsagePayload> usagePayload, RateCardPayload rateCardPayload, TraceWriter log)
{
double totalSpend = 0;
if (usagePayload == null || rateCardPayload == null)
{
return(totalSpend);
}
foreach (var usageGroup in usagePayload.SelectMany(usage => usage.Value).GroupBy(usage => $"{usage.Properties.MeterCategory} {usage.Properties.MeterSubCategory}"))
{
double typeSpend = 0;
foreach (var usage in usageGroup)
{
var meter = rateCardPayload.Meters.FirstOrDefault(iterMeter => iterMeter.MeterId.Equals(usage.Properties.MeterId));
if (meter != null)
{
typeSpend += CalcMeterPerUsage(usage, meter, log);
}
else
{
log.Info($"could not find meter for usage: {usage.Name}");
}
}
log.Info($"total spend in month on {usageGroup.Key} is {typeSpend}");
totalSpend += typeSpend;
}
log.Info($"total spend in month {totalSpend}");
return(totalSpend);
}
/// <summary>
/// Method to get project rates.
/// </summary>
/// <param name="rates">Rate card information.</param>
/// <param name="projects">Projects collection.</param>
/// <returns>Project rates.</returns>
public static IEnumerable <ProjectRate> GetProjectRates(RateCardPayload rates, IEnumerable <ProjectUsage> projects)
{
IEnumerable <ProjectRate> ratesList = from p in projects
select new ProjectRate(
p.ProjectName, GetMeterIDRates(p.MeterIdTotals, rates));
return(ratesList);
}
/// <summary>
/// Method to get Rate Card object.
/// </summary>
/// <returns>Rate Card object.</returns>
public static RateCardPayload GetRateCardPayload()
{
RateCardPayload resourceRatesRootObject =
JsonConvert.DeserializeObject <RateCardPayload>(
File.ReadAllText(@"C:\Users\MoinakBandyopadhyay\Desktop\response.json"));
return(resourceRatesRootObject);
}
/// <summary>
/// Method to get included quantity for meter Id
/// </summary>
/// <param name="rates">rates information</param>
/// <param name="meterId">meter id</param>
/// <returns>included quantity</returns>
public static double GetIncludedQuantityForMeterID(RateCardPayload rates, string meterId)
{
List <Resource> resources = rates.Meters;
IEnumerable <double> includedquantity = from r in resources
where r.MeterId == meterId
select r.IncludedQuantity;
return(includedquantity.FirstOrDefault());
}
/// <summary>
/// method to get Resource Name by Meter Id
/// </summary>
/// <param name="rates">rates information</param>
/// <param name="meterId">Meter id</param>
/// <returns>Resource name</returns>
public static string GetResourceNameByMeterId(RateCardPayload rates, string meterId)
{
List <Resource> resources = rates.Meters;
IEnumerable <string> rate = from r in resources
where r.MeterId == meterId
select r.MeterName;
return(rate.FirstOrDefault());
}
/// <summary>
/// Method to get Resource rates.
/// </summary>
/// <param name="rates">Rate card information.</param>
/// <param name="resourceUsage">Resource usage collection.</param>
/// <returns>Resource rate collection.</returns>
public static IEnumerable <ResourceRate> GetResourceRates(
RateCardPayload rates,
IEnumerable <ResourceUsage> resourceUsage)
{
IEnumerable <ResourceRate> ratesList = from r in resourceUsage
select new ResourceRate(
r.ResourceName, GetMeterIDRates(r.MeterIdTotals, rates));
return(ratesList);
}
/// <summary>
/// Method to get rates for Meter ID.
/// </summary>
/// <param name="rates">Rates information.</param>
/// <param name="meterId">Meter id.</param>
/// <returns>Meter rates.</returns>
public static Dictionary <double, double> GetRatesForMeterID(RateCardPayload rates, string meterId)
{
List <Resource> resources = rates.Meters;
Resource r = resources.Find(x => x.MeterId == meterId);
if (r == null)
{
r = GetUndefinedResource();
}
return(r.MeterRates);
}
public static async Task <RateCardPayload> GetRateCard(string token, string billingService, string subscriptionId)
{
/*Setup API call to RateCard API
* Callouts:
* See the App.config file for all AppSettings key/value pairs
* You can get a list of offer numbers from this URL: http://azure.microsoft.com/en-us/support/legal/offer-details/
* You can configure an OfferID for this API by updating 'MS-AZR-{Offer Number}'
* The RateCard Service/API is currently in preview; please use "2015-06-01-preview" or "2016-08-31-preview" for api-version (see https://msdn.microsoft.com/en-us/library/azure/mt219005 for details)
* Please see the readme if you are having problems configuring or authenticating: https://github.com/Azure-Samples/billing-dotnet-ratecard-api
*/
// Build up the HttpWebRequest
string requestURL = String.Format("{0}/{1}/{2}/{3}",
billingService,
"subscriptions",
subscriptionId,
"providers/Microsoft.Commerce/RateCard?api-version=2016-08-31-preview&$filter=OfferDurableId eq 'MS-AZR-0063P' and Currency eq 'GBP' and Locale eq 'en-GB' and RegionInfo eq 'GB'");
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestURL);
// Add the OAuth Authorization header, and Content Type header
request.Headers["Authorization"] = "Bearer " + token;
request.ContentType = "application/json";
RateCardPayload payload = new RateCardPayload();
// Call the RateCard API
try
{
// Call the REST endpoint
WebResponse response = await request.GetResponseAsync();
Stream receiveStream = response.GetResponseStream();
// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
var rateCardResponse = readStream.ReadToEnd();
// Convert the Stream to a strongly typed RateCardPayload object.
// You can also walk through this object to manipulate the individuals member objects.
payload = JsonConvert.DeserializeObject <RateCardPayload>(rateCardResponse);
response.Dispose();
readStream.Dispose();
}
catch (Exception e)
{
Console.WriteLine(String.Format("{0} \n\n{1}", e.Message, e.InnerException != null ? e.InnerException.Message : ""));
Console.ReadLine();
}
return(payload);
}
public static RateCardPayload GetRateCardInfo(string restUrl, Guid orgId)
{
HttpWebResponse httpWebResponse = AzureResourceManagerUtil.RateCardRestApiCall(restUrl, orgId);
try {
if (httpWebResponse == null)
{
Trace.TraceWarning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
Trace.TraceWarning($"{nameof(httpWebResponse)} == null");
Trace.TraceWarning($" {nameof(GetRateCardInfo)}({nameof(restUrl)}, {nameof(orgId)})");
Trace.TraceWarning($" {nameof(restUrl)}: {restUrl}");
Trace.TraceWarning($" {nameof(orgId)}: {orgId}");
// throw exception to start from scretch and retry.
//throw new Exception("Possible reason: Bad request (400), Forbidden (403) to access. Server busy. Client blacklisted.");
}
else
{
// look response codes @ https://msdn.microsoft.com/en-us/library/azure/mt219001.aspx
// see: https://msdn.microsoft.com/en-us/library/azure/mt219004.aspx
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Trace.TraceInformation($"Received Rest Call Response: {nameof(HttpStatusCode.OK)}. Processing...");
string streamContent;
using (Stream receiveStream = httpWebResponse.GetResponseStream()) {
// Pipes the stream to a higher level stream reader with the required encoding format.
using (StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8)) {
streamContent = readStream.ReadToEnd();
}
}
RateCardPayload rateCardPayload = JsonConvert.DeserializeObject <RateCardPayload>(streamContent);
return(rateCardPayload);
}
else if (httpWebResponse.StatusCode == HttpStatusCode.Accepted)
{
Trace.TraceWarning($"Data not ready. {nameof(HttpStatusCode.Accepted)}. Not capable of handling this.");
}
else
{
Trace.TraceWarning("NEW RESPONSE TYPE. HANDLE THIS - GetRateCardInfo!");
Trace.TraceWarning($"code:{httpWebResponse.StatusCode} desc:{httpWebResponse.StatusDescription}");
}
}
return(null);
} finally {
httpWebResponse?.Dispose();
}
}
/// <summary>
/// Method to get Meter id rates.
/// </summary>
/// <param name="meterIdTotals">Meter id totals.</param>
/// <param name="rates">Rate card information.</param>
/// <returns>Rated usage.</returns>
public static IEnumerable <RatedUsage> GetMeterIDRates(
IEnumerable <MeterIDTotal> meterIdTotals,
RateCardPayload rates)
{
IEnumerable <RatedUsage> ratesList = from m in meterIdTotals
select new RatedUsage(
m.MeterId,
m.ResourceTotals,
GetRatesForMeterID(rates, m.MeterId),
ComputeRatedUsagePerMeter(GetRatesForMeterID(rates, m.MeterId), GetRatableUsage(m.ResourceTotals, GetIncludedQuantityForMeterID(rates, m.MeterId))),
GetMeterSubCategoryByMeterId(rates, m.MeterId),
GetServiceByMeterId(rates, m.MeterId),
GetResourceNameByMeterId(rates, m.MeterId));
return(ratesList);
}
private async Task <List <Resource> > GetPrice(string token, string subscription)
{
var filter = "&$filter=OfferDurableId eq 'MS-AZR-0003p' and Currency eq 'KRW' and Locale eq 'en-us' and RegionInfo eq 'KR'";
var uri = "/subscriptions/" + subscription + "/providers/Microsoft.Commerce/RateCard?api-version=2015-06-01-preview" + filter;
using (var client = new HttpClient())
{
client.BaseAddress = new Uri("https://management.azure.com");
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Add("Authorization", "Bearer " + token);
var resultContent = client.GetStringAsync(uri);
RateCardPayload payload = JsonConvert.DeserializeObject <RateCardPayload>(await resultContent);
return(payload.Meters);
}
}
public static RateCardPayload GetRateCardInfo(string restURL, string orgID)
{
HttpWebResponse httpWebResponse = null;
httpWebResponse = AzureResourceManagerUtil.RateCardRestApiCall(restURL, orgID);
if (httpWebResponse == null)
{
Console.WriteLine("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
Console.WriteLine("httpWebResponse == null");
Console.WriteLine(" GetRateCardInfo(string restURL, string orgID)");
Console.WriteLine(" restURL: {0}", restURL);
Console.WriteLine(" orgID: {0}", orgID);
// throw exception to start from scretch and retry.
//throw new Exception("Possible reason: Bad request (400), Forbidden (403) to access. Server busy. Client blacklisted.");
}
else
{
// look response codes @ https://msdn.microsoft.com/en-us/library/azure/mt219001.aspx
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Console.WriteLine("Received Rest Call Response: HttpStatusCode.OK. Processing...");
Stream receiveStream = httpWebResponse.GetResponseStream();
// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string streamContent = readStream.ReadToEnd();
RateCardPayload rateCardPayload = JsonConvert.DeserializeObject <RateCardPayload>(streamContent);
return(rateCardPayload);
}
else if (httpWebResponse.StatusCode == HttpStatusCode.Accepted)
{
Console.WriteLine("Data not ready. HttpStatusCode.Accepted. Not capable of handling this.");
}
else
{
Console.WriteLine("NEW RESPONSE TYPE. HANDLE THIS - GetRateCardInfo!");
Console.WriteLine("code:{0} desc:{1}", httpWebResponse.StatusCode, httpWebResponse.StatusDescription);
}
}
return(null);
}
/// <summary>
/// Method to collect azure usage records from Azure APIs.
/// </summary>
/// <returns>Collection of Usage records.</returns>
private IEnumerable <UsageInfoModel> GetUsageInfo()
{
// Loop for every subscription
foreach (var subscription in this.subscriptions)
{
// Get usage details for subscription
RateCardPayload rates = AzureResourceManagerUtil.GetRates(subscription.Id, org.Id, authResult);
List <UsageAggregate> usageList = AzureResourceManagerUtil.GetUsage(
subscription.Id,
org.Id,
authResult,
this.startDate,
this.endDate);
IEnumerable <string> projects = AzureResourceManagerUtil.GetUniqueProjects(usageList);
subscription.Projects = projects;
IEnumerable <ProjectUsage> projectTotals =
AzureResourceManagerUtil.GetProjectIDTotals(usageList, projects);
subscription.ProjectTotals = projectTotals;
IEnumerable <ProjectRate> projectRates = AzureResourceManagerUtil.GetProjectRates(
rates,
projectTotals);
subscription.ProjectRates = projectRates;
IEnumerable <ProjectEstimate> projectEstimateTotals =
AzureResourceManagerUtil.RatedEstimateForProject(projectRates);
subscription.ProjectEstimateTotals = projectEstimateTotals;
IEnumerable <string> resourceList = AzureResourceManagerUtil.GetResources(usageList);
IEnumerable <ResourceUsage> resourceUsages =
AzureResourceManagerUtil.GetResourceUsage(usageList, resourceList);
subscription.ResourceTotals = AzureResourceManagerUtil.GetResourceTotals(resourceList, usageList);
IEnumerable <ResourceRate> resourceRates =
AzureResourceManagerUtil.GetResourceRates(rates, resourceUsages);
IEnumerable <ResourceEstimate> resourceEstimates =
AzureResourceManagerUtil.RatedEstimateForResource(resourceRates);
subscription.ResourceEstimates = resourceEstimates;
subscription.MeterIdTotals = AzureResourceManagerUtil.GetMeterIdTotals(usageList);
subscription.ServiceTotals = AzureResourceManagerUtil.GetServiceTotals(usageList);
var result = JsonConvert.SerializeObject(subscription.MeterIdTotals); // Input to graphing solution
subscription.UsageDetails = result;
IEnumerable <RatedUsage> ratedUsageList =
AzureResourceManagerUtil.GetMeterIDRates(
AzureResourceManagerUtil.GetMeterIdTotals(usageList), rates);
subscription.ratedUsage = ratedUsageList;
subscription.RatedEstimate = AzureResourceManagerUtil.RatedEstimate(ratedUsageList);
foreach (var usg in usageList)
{
double usgtot =
AzureResourceManagerUtil.ComputeRatedUsagePerMeter(
AzureResourceManagerUtil.GetRatesForMeterID(rates, usg.properties.meterId),
usg.properties.quantity);
this.usageInfo.Add(new UsageInfoModel()
{
OrganizationId = org.Id,
SubceriptionId = subscription.Id,
UsageStartTime = usg.properties.usageStartTime,
UsageEndTime = usg.properties.usageEndTime,
MeteredRegion = usg.properties.infoFields.MeteredRegion,
MeterName = usg.properties.meterName,
MeteredService = usg.properties.infoFields.MeteredService,
MeteredServiceType = usg.properties.infoFields.MeteredServiceType,
MeterCategory = usg.properties.meterCategory,
MeterSubCategory = usg.properties.meterSubCategory,
UserProject = usg.properties.infoFields.Project,
Quantity = usg.properties.quantity,
ItemTotal = usgtot
});
}
}
return(this.usageInfo);
}
private static double SpendDuringCurrentMonth(string token, string armBillingService,
string subscriptionId, string subscriptionType, TraceWriter log, int currentMonth) =>
Math.Round(
Calculate(UsagePayloads.GetUsagePayLoads(token, armBillingService, subscriptionId, log, currentMonth),
RateCardPayload.GetRateCardPayLoad(token, armBillingService, subscriptionId, subscriptionType, log),
log), 1);
public static void ProcessQueueMessage([QueueTrigger("billingdatarequests")] BillingRequest br)
{
Console.WriteLine("Start webjob process. SubscriptionID: {0}", br.SubscriptionId);
int retriesLeft = Convert.ToInt32(ConfigurationManager.AppSettings["ida:RetryCountToProcessMessage"].ToString());
while (retriesLeft > 0)
{
--retriesLeft;
if (retriesLeft < 1)
{
Console.WriteLine("Finished internal retries, throwing exception. Time:{0}", DateTime.UtcNow.ToString());
throw new Exception();
}
Console.WriteLine("Start time:{0} Retries Left: {1}", DateTime.UtcNow.ToString(), retriesLeft);
try
{
//Fetch RateCard information First
string rateCardURL = AzureResourceManagerUtil.GetRateCardRestApiCallURL(br.SubscriptionId,
ConfigurationManager.AppSettings["ida:OfferCode"].ToString(),
ConfigurationManager.AppSettings["ida:Currency"].ToString(),
ConfigurationManager.AppSettings["ida:Locale"].ToString(),
ConfigurationManager.AppSettings["ida:RegionInfo"].ToString());
Console.WriteLine("Request cost info from RateCard service.");
RateCardPayload rateCardInfo = GetRateCardInfo(rateCardURL, br.OrganizationId);
if (rateCardInfo == null)
{
Console.WriteLine("Problem receiving cost info occured - see log for details.");
continue;
}
else
{
Console.WriteLine("Received cost info: " + rateCardInfo.ToString());
}
// if granularity=hourly then report up to prev. hour,
// if granularity=daily then report up to prev. day. Othervise will get 400 error
//DateTime sdt = DateTime.Now.AddYears(-3);
//DateTime edt = DateTime.Now.AddDays(-1);
string restURL = AzureResourceManagerUtil.GetBillingRestApiCallURL(br.SubscriptionId, false, true, br.StartDate, br.EndDate);
Console.WriteLine("Request usage data from Billing service.");
List <UsageRecord> urs = GetUsageDetails(restURL, br.OrganizationId, rateCardInfo);
Console.WriteLine("Received record count: {0}", urs.Count);
Console.WriteLine("Insert usage data into SQL Server.");
InsertIntoSQLDB(urs);
break;
}
catch (Exception e)
{
Console.WriteLine("Exception: ProcessQueueMessage->e.Message: " + e.Message);
if (retriesLeft == 0)
{
throw;
}
}
Console.WriteLine("Sleeping in ProcessQueueMessage while loop for 5 min. DateTime: {0}", DateTime.Now.ToString());
Thread.Sleep(1000 * 60 * 5);
} // while
Commons.Utils.UpdateSubscriptionStatus(br.SubscriptionId, DataGenStatus.Completed, DateTime.UtcNow);
Console.WriteLine("Complete webjob process. SubscriptionID: {0}", br.SubscriptionId);
}
public static List <UsageRecord> GetUsageDetails(string restURL, string orgID, RateCardPayload rateCardInfo)
{
string nextLink = "";
List <UsageRecord> usageRecords = new List <UsageRecord>();
do
{
HttpWebResponse httpWebResponse = null;
if (nextLink != "")
{
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(nextLink, orgID);
}
else
{
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(restURL, orgID);
}
if (httpWebResponse == null)
{
Console.WriteLine("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
Console.WriteLine("httpWebResponse == null");
Console.WriteLine(" GetUsageDetails(string restURL, string orgID)");
Console.WriteLine(" restURL: {0}", restURL);
Console.WriteLine(" orgID: {0}", orgID);
// throw exception to start from scretch and retry.
//throw new Exception("Possible reason: Bad request (400), Forbidden (403) to access. Server busy. Client blacklisted.");
}
else
{
// look response codes @ https://msdn.microsoft.com/en-us/library/azure/mt219001.aspx
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Console.WriteLine("Received Rest Call Response: HttpStatusCode.OK. Processing...");
Stream receiveStream = httpWebResponse.GetResponseStream();
// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string streamContent = readStream.ReadToEnd();
UsagePayload usagePayload = JsonConvert.DeserializeObject <UsagePayload>(streamContent);
foreach (UsageAggregate ua in usagePayload.value)
{
//Handle adding cost in
var ur = new UsageRecord(ua);
try {
var meterInfo = rateCardInfo.Meters.Where(p => p.MeterId == ur.meterId).SingleOrDefault();
ur.cost = meterInfo.MeterRates["0"] * Convert.ToDecimal(ur.quantity);
if (ur.cost < 0.01M)
{
ur.cost = 0;
}
} catch (Exception ex)
{
Console.WriteLine("Exception trying to apply cost info for meter: " + ur.meterId);
}
usageRecords.Add(ur);
}
ContinuationToken contToken = JsonConvert.DeserializeObject <ContinuationToken>(streamContent);
if (contToken.nextLink != null)
{
nextLink = contToken.nextLink;
}
else
{
nextLink = "";
}
}
else if (httpWebResponse.StatusCode == HttpStatusCode.Accepted)
{
Console.WriteLine("Data not ready. HttpStatusCode.Accepted. Waiting 6 min. now: {0}", DateTime.Now.ToString());
Thread.Sleep(1000 * 60 * 6); // wait a bit to have data get prepared by azure
nextLink = restURL; // set next link to same URL for second call
}
else
{
Console.WriteLine("NEW RESPONSE TYPE. HANDLE THIS!");
Console.WriteLine("code:{0} desc:{1}", httpWebResponse.StatusCode, httpWebResponse.StatusDescription);
}
}
} while (nextLink != "");
return(usageRecords);
}
public static List<UsageRecord> GetUsageDetails(string restURL, string orgID, RateCardPayload rateCardInfo)
{
string nextLink = "";
List<UsageRecord> usageRecords = new List<UsageRecord>();
do
{
HttpWebResponse httpWebResponse = null;
if (nextLink != "")
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(nextLink, orgID);
else
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(restURL, orgID);
if (httpWebResponse == null)
{
Console.WriteLine("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
Console.WriteLine("httpWebResponse == null");
Console.WriteLine(" GetUsageDetails(string restURL, string orgID)");
Console.WriteLine(" restURL: {0}", restURL);
Console.WriteLine(" orgID: {0}", orgID);
// throw exception to start from scretch and retry.
//throw new Exception("Possible reason: Bad request (400), Forbidden (403) to access. Server busy. Client blacklisted.");
}
else
{
// look response codes @ https://msdn.microsoft.com/en-us/library/azure/mt219001.aspx
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Console.WriteLine("Received Rest Call Response: HttpStatusCode.OK. Processing...");
Stream receiveStream = httpWebResponse.GetResponseStream();
// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8);
string streamContent = readStream.ReadToEnd();
UsagePayload usagePayload = JsonConvert.DeserializeObject<UsagePayload>(streamContent);
foreach (UsageAggregate ua in usagePayload.value)
{
//Handle adding cost in
var ur = new UsageRecord(ua);
try {
var meterInfo = rateCardInfo.Meters.Where(p => p.MeterId == ur.meterId).SingleOrDefault();
ur.cost = meterInfo.MeterRates["0"] * Convert.ToDecimal(ur.quantity);
if (ur.cost < 0.01M)
ur.cost = 0;
} catch (Exception ex)
{
Console.WriteLine("Exception trying to apply cost info for meter: " + ur.meterId);
}
usageRecords.Add(ur);
}
ContinuationToken contToken = JsonConvert.DeserializeObject<ContinuationToken>(streamContent);
if (contToken.nextLink != null)
nextLink = contToken.nextLink;
else
nextLink = "";
}
else if (httpWebResponse.StatusCode == HttpStatusCode.Accepted)
{
Console.WriteLine("Data not ready. HttpStatusCode.Accepted. Waiting 6 min. now: {0}", DateTime.Now.ToString());
Thread.Sleep(1000 * 60 * 6); // wait a bit to have data get prepared by azure
nextLink = restURL; // set next link to same URL for second call
}
else
{
Console.WriteLine("NEW RESPONSE TYPE. HANDLE THIS!");
Console.WriteLine("code:{0} desc:{1}", httpWebResponse.StatusCode, httpWebResponse.StatusDescription);
}
}
} while (nextLink != "");
return usageRecords;
}
public static List <ResourceGroupAggregate> CreateResourceGroupAggregates(UsagePayload usagePayload, RateCardPayload rateCardPayload)
{
List <ResourceGroupAggregate> resourceGroupAggregates = new List <ResourceGroupAggregate>();
// Get distinct resource groups
foreach (UsageAggregate usage in usagePayload.Value)
{
if (usage.Properties.InstanceData.MicrosoftResources.ResourceList.ContainsKey("resourceGroups"))
{
// Get resource group
ResourceGroupAggregate resourceGroupAggregate = resourceGroupAggregates.SingleOrDefault(x => x.ResourceGroupName.ToLower() == usage.Properties.InstanceData.MicrosoftResources.ResourceList["resourceGroups"].ToLower());
// Add resource group if it doesn't already exist
if (resourceGroupAggregate == null)
{
resourceGroupAggregate = new ResourceGroupAggregate();
resourceGroupAggregate.ResourceGroupName = usage.Properties.InstanceData.MicrosoftResources.ResourceList["resourceGroups"];
resourceGroupAggregates.Add(resourceGroupAggregate);
resourceGroupAggregate = resourceGroupAggregates.SingleOrDefault(x => x.ResourceGroupName == usage.Properties.InstanceData.MicrosoftResources.ResourceList["resourceGroups"]);
}
// Get resource item
ResourceItem resourceItem = resourceGroupAggregate.ResourceItems.SingleOrDefault(x => x.MeterName == usage.Properties.MeterName);
// Add resource item if it doesn't already exist
if (resourceItem == null)
{
resourceItem = new ResourceItem();
resourceItem.MeterCategory = usage.Properties.MeterCategory;
resourceItem.MeterName = usage.Properties.MeterName;
resourceGroupAggregate.ResourceItems.Add(resourceItem);
resourceItem = resourceGroupAggregate.ResourceItems.SingleOrDefault(x => x.MeterName == usage.Properties.MeterName);
}
ResourceDateItem resourceDateItem = new ResourceDateItem();
resourceDateItem.Quantity = usage.Properties.Quantity;
resourceDateItem.Cost = rateCardPayload.Meters.Where(x => x.MeterId == usage.Properties.MeterId).FirstOrDefault().MeterRates[0] * usage.Properties.Quantity;
resourceDateItem.UsageStartTime = DateTime.Parse(usage.Properties.UsageStartTime);
resourceDateItem.UsageEndTime = DateTime.Parse(usage.Properties.UsageEndTime);
resourceItem.ResourceDateItems.Add(resourceDateItem);
}
}
return(resourceGroupAggregates);
}
public static List <UsageRecord> GetUsageDetails(string restUrl, Guid orgId, RateCardPayload rateCardInfo)
{
string nextLink = "";
List <UsageRecord> usageRecords = new List <UsageRecord>();
do
{
HttpWebResponse httpWebResponse = null;
try {
if (nextLink != "")
{
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(nextLink, orgId);
}
else
{
httpWebResponse = AzureResourceManagerUtil.BillingRestApiCall(restUrl, orgId);
}
if (httpWebResponse == null)
{
Trace.TraceWarning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
Trace.TraceWarning($"{nameof(httpWebResponse)} == null");
Trace.TraceWarning($" {nameof(GetUsageDetails)}({nameof(restUrl)}, {nameof(orgId)})");
Trace.TraceWarning($" {nameof(restUrl)}: {restUrl}");
Trace.TraceWarning($" {nameof(orgId)}: {orgId}");
// throw exception to start from scretch and retry.
//throw new Exception("Possible reason: Bad request (400), Forbidden (403) to access. Server busy. Client blacklisted.");
}
else
{
// look response codes @ https://msdn.microsoft.com/en-us/library/azure/mt219001.aspx
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Trace.TraceInformation($"Received Rest Call Response: {nameof(HttpStatusCode.OK)}. Processing...");
string streamContent;
using (Stream receiveStream = httpWebResponse.GetResponseStream()) {
// Pipes the stream to a higher level stream reader with the required encoding format.
using (StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8)) {
streamContent = readStream.ReadToEnd();
}
}
UsagePayload usagePayload = JsonConvert.DeserializeObject <UsagePayload>(streamContent);
foreach (UsageAggregate ua in usagePayload.Value)
{
// handle adding cost in
var usageRecord = new UsageRecord(ua);
try {
var meterInfo = rateCardInfo.Meters.Where(p => p.MeterId == usageRecord.MeterId).SingleOrDefault();
if (meterInfo.MeterRates.Count > 1)
{
Trace.TraceWarning("Multiple rates for meter: " + usageRecord.MeterId);
}
usageRecord.Cost = (double)meterInfo.MeterRates["0"] * usageRecord.Quantity;
//if (usageRecord.cost < 0.01) usageRecord.cost = 0; // TODO: usage cost is definitelly NOT rounded like this
} catch (Exception ex) {
Trace.TraceError("Exception trying to apply cost info for meter: " + usageRecord.MeterId);
}
usageRecords.Add(usageRecord);
}
ContinuationToken contToken = JsonConvert.DeserializeObject <ContinuationToken>(streamContent);
nextLink = contToken.NextLink ?? "";
}
else if (httpWebResponse.StatusCode == HttpStatusCode.Accepted)
{
Trace.TraceWarning($"Data not ready. {nameof(HttpStatusCode.Accepted)}. Waiting 6 min. now: {DateTime.UtcNow}");
Thread.Sleep(1000 * 60 * 6); // wait a bit to have data get prepared by azure
nextLink = restUrl; // set next link to same URL for second call
}
else
{
Trace.TraceWarning("NEW RESPONSE TYPE. HANDLE THIS!");
Trace.TraceWarning($"code:{httpWebResponse.StatusCode} desc:{httpWebResponse.StatusDescription}");
}
}
} finally {
httpWebResponse?.Dispose();
}
} while (nextLink != "");
return(usageRecords);
}
public static void ProcessQueueMessage([QueueTrigger("billingdatarequests")] BillingRequest billingRequest, TextWriter logWriter = null)
{
if (logWriter != null)
{
TextWriterTraceListener traceListener = new TextWriterTraceListener(logWriter, "LogWriter");
Trace.Listeners.Remove("LogWriter");
Trace.Listeners.Add(traceListener);
Trace.TraceInformation("Azure WebJob Log Writer configured");
}
Trace.TraceInformation($"WebJob process started. {nameof(billingRequest.SubscriptionId)}: {billingRequest.SubscriptionId}");
int retriesLeft = Convert.ToInt32(RetryCountToProcessMessage);
Exception lastException = null;
while (retriesLeft > 0)
{
--retriesLeft;
if (retriesLeft < 1)
{
Trace.TraceInformation($"Finished internal retries, time:{DateTime.UtcNow}");
if (lastException != null)
{
throw lastException;
}
else
{
return;
}
}
Trace.TraceInformation($"Start time:{DateTime.UtcNow}, retries Left: {retriesLeft}");
try {
//Fetch RateCard information First
string rateCardUrl = AzureResourceManagerUtil.GetRateCardRestApiCallURL(billingRequest.SubscriptionId, OfferCode, Currency, Locale, RegionInfo);
Trace.TraceInformation("Request cost info from RateCard service.");
RateCardPayload rateCardInfo = GetRateCardInfo(rateCardUrl, billingRequest.OrganizationId);
if (rateCardInfo == null)
{
Trace.TraceWarning("Problem receiving cost info occured - see log for details.");
continue;
}
else
{
Trace.TraceInformation("Received cost info: " + rateCardInfo.ToString());
}
// if granularity=hourly then report up to prev. hour,
// if granularity=daily then report up to prev. day. Othervise will get 400 error
//DateTime sdt = DateTime.UtcNow.Date.AddYears(-3);
//DateTime edt = DateTime.UtcNow.Date.AddDays(-1);
// see: https://msdn.microsoft.com/en-us/library/azure/mt219004.aspx
string restUrl = AzureResourceManagerUtil.GetBillingRestApiCallUrl(billingRequest.SubscriptionId, true, true, billingRequest.StartDate, billingRequest.EndDate);
Trace.TraceInformation("Request usage data from Billing service.");
var usageRecords = GetUsageDetails(restUrl, billingRequest.OrganizationId, rateCardInfo);
Trace.TraceInformation($"Received record count: {usageRecords.Count}");
if (usageRecords.Count > 0)
{
Trace.TraceInformation("Inserting usage records into SQL database.");
Task <int> task = InsertIntoSqlDbAsync(usageRecords, billingRequest.SubscriptionId, billingRequest.StartDate, billingRequest.EndDate);
int recordCount = task.GetAwaiter().GetResult();
Trace.TraceInformation($"Total {recordCount} usage record(s) inserted.");
}
else
{
Trace.TraceInformation("No usage data found.");
}
break;
} catch (Exception e) {
Trace.TraceError($"Exception: {nameof(ProcessQueueMessage)} -> e.Message: " + e.Message);
lastException = e;
if (retriesLeft == 0)
{
throw;
}
}
Trace.TraceInformation($"Sleeping in {nameof(ProcessQueueMessage)} while loop for 5 min. DateTime: {DateTime.UtcNow}");
Thread.Sleep(1000 * 60 * 5);
} // while
Utils.UpdateSubscriptionStatus(billingRequest.SubscriptionId, DataGenStatus.Completed, DateTime.UtcNow);
Trace.TraceInformation($"WebJob process completed. SubscriptionId: {billingRequest.SubscriptionId}");
}