public static void WriteReport(ConsumptionAnalysisReport report, string outputFile) { using var writer = File.CreateText(outputFile); var resources = report.GetResources(); writer.WriteLine( "SubscriptionId," + "Resource name," + "Resource ID," + "Service Type," + "Cost," + "Currency," + "Billing Start," + "Billing End," + "Actual resource deletion date," + "DeleteOperationId," + "Overage"); foreach (var billedResource in resources) { writer.WriteLine($"{billedResource.SubscriptionId}," + $"{billedResource.InstanceName.Replace(',', '_')}," + $"{billedResource.Id.Replace(',', '_')}," + $"{billedResource.ConsumedService}," + $"{billedResource.PretaxCost}," + $"{billedResource.Currency}," + $"{billedResource.UsageStart}," + $"{billedResource.UsageEnd}," + $"{billedResource.ActivityDeleted}," + $"{billedResource.DeleteOperationId}," + $"{billedResource.Overage}"); } }
public async Task <ConsumptionAnalysisReport> AnalyzeConsumptionForDeletedResources(IList <UsageDetail> usage, bool onlyWithOverages) { var report = new ConsumptionAnalysisReport(); // Get resources with non - zero costs var processingPool = new ConcurrentQueue <ProcessingTask>(usage .GroupBy(r => r.InstanceId.ToLowerInvariant()) .Select(u => new { ResourceId = u.First().InstanceId, Costs = u.Sum(c => c.PretaxCost) }) .Where(p => p.Costs > 0) .Select(t => new ProcessingTask { ResourceId = t.ResourceId })); var totalResources = processingPool.Count; Log.Information( $"Subscription ${_subscriptionId}. Retrieved {totalResources} resources with non-zero billing"); // Running processing in parallel threads var processingThreads = processingPool.Count > MaxProcessingThreads ? MaxProcessingThreads : processingPool.Count; var tasks = new List <Task>(processingThreads); var processedCount = 0; for (var i = 0; i < processingThreads; i++) { var task = Task.Run(() => { ProcessingTask task = null; while (processingPool.TryDequeue(out task)) { try { if (task.Exceptions.Count > ProcessingRetryCount) { Log.Warning($"Subscription ${_subscriptionId}. Failed to process {task.ResourceId}"); continue; } // Skip existing resources var resourceExists = _resourceProvider.IsResourceExists(task.ResourceId).Result; if (resourceExists == null) { Log.Information( $"Item skipped. Billing Item {task.ResourceId} is not actually a resource"); continue; } if (resourceExists.Value) { Log.Information($"Item skipped. Item {task.ResourceId} exists at the moment"); continue; } // Delete event is missing for some resources, so use resource group deletion date var deleteActivity = _activityLogProvider.GetResourceDeletionDate(task.ResourceId).Result ?? _activityLogProvider .GetResourceGroupDeletionDate( GetResourceGroupName(task.ResourceId)).Result; if (deleteActivity == null || onlyWithOverages && !usage.Any(r => r.InstanceId == task.ResourceId && r.UsageStart > deleteActivity?.EventTimestamp)) { processedCount++; continue; } // Calculate overage as sum of billing records for dates past deletion report.AddResource(new BilledResources { SubscriptionId = usage.First(r => r.InstanceId == task.ResourceId).SubscriptionGuid, Currency = usage.First(r => r.InstanceId == task.ResourceId).Currency, Id = task.ResourceId, InstanceName = usage.First(r => r.InstanceId == task.ResourceId).InstanceName, PretaxCost = usage.Where(r => r.InstanceId == task.ResourceId).Sum(u => u.PretaxCost), ConsumedService = usage.First(r => r.InstanceId == task.ResourceId).ConsumedService, SubscriptionName = usage.First(r => r.InstanceId == task.ResourceId).SubscriptionName, UsageStart = usage.Where(r => r.InstanceId == task.ResourceId).Min(u => u.UsageStart), UsageEnd = usage.Where(r => r.InstanceId == task.ResourceId).Max(u => u.UsageEnd), ActivityDeleted = deleteActivity?.EventTimestamp, DeleteOperationId = deleteActivity?.OperationId, Overage = deleteActivity == null ? 0 : usage.Where(r => r.InstanceId == task.ResourceId && r.UsageStart > deleteActivity.EventTimestamp) .Sum(o => o.PretaxCost) }); processedCount++; } catch (Exception exception) { // Activity API sometimes fails with timeouts, need to retry Log.Error(exception, $"Exception while processing resource {task.ResourceId}, retrying"); task.Exceptions.Add(exception); processingPool.Enqueue(task); } } }).ContinueWith(t => { if (!t.IsCompletedSuccessfully) { Log.Error(t.Exception?.GetBaseException(), "Analysis thread crashed"); } }); tasks.Add(task); } using var timer = new Timer(data => Log.Information( $"Subscription {_subscriptionId}. Processed ... {processedCount} of {totalResources}"), null, 0, 10000); await Task.WhenAll(tasks); return(report); }