public static async Task ProcessSecurityAsync(
[QueueTrigger(OperationConstants.SecurityQueueName, Connection = "StorageConnectionString")] SecurityDetails data,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(Alert),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <Alert> securityAlertRepository,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(SecureScore),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <SecureScore> secureScoreRepository,
[SecureScore(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Period = 1,
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <SecureScore> scores,
[SecurityAlerts(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <Alert> alerts,
TraceWriter log)
{
log.Info($"Attempting to process data for {data.Customer.Id}");
if (data.Customer.ProcessException != null)
{
log.Warning($"Unable to process {data.Customer.Id} please check the customer's last exception for more information.");
return;
}
if (scores?.Count > 0)
{
log.Info($"Importing {scores.Count} Secure Score entries for {data.Customer.Id}");
await secureScoreRepository.AddOrUpdateAsync(scores).ConfigureAwait(false);
}
if (alerts?.Count > 0)
{
log.Info($"Importing {alerts.Count} security alert entries for {data.Customer.Id}");
await securityAlertRepository.AddOrUpdateAsync(alerts).ConfigureAwait(false);
}
log.Info($"Successfully process data for {data.Customer.Id}");
}
public static async Task ImportControlsAsync(
[TimerTrigger("0 0 10 * * *")] TimerInfo timerInfo,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(ControlListEntry),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <ControlListEntry> repository,
TraceWriter log)
{
CsvReader reader = null;
IEnumerable <ControlListEntry> entries;
try
{
log.Info("Importing Office 365 Secure Score controls details...");
if (timerInfo.IsPastDue)
{
log.Info("Execution of the function is starting behind schedule.");
}
using (StreamReader streamReader = new StreamReader(
Assembly.GetExecutingAssembly().GetManifestResourceStream(ControlListEmbeddedResource)))
{
reader = new CsvReader(streamReader);
reader.Configuration.RegisterClassMap <ControlListEntryClassMap>();
entries = reader.GetRecords <ControlListEntry>();
await repository.AddOrUpdateAsync(entries).ConfigureAwait(false);
}
log.Info("Successfully import Office 365 Secure Score controls.");
}
finally
{
entries = null;
reader?.Dispose();
}
}
public async Task <IActionResult> AddNew(
[Bind("AppEndpoint,EnvironmentType,FriendlyName,Id,PartnerCenterEndpoint,ProcessAzureUsage")] EnvironmentDetail environment)
{
if (ModelState.IsValid)
{
environment.Modified = DateTimeOffset.Now;
if (!string.IsNullOrEmpty(environment.AppEndpoint?.ApplicationSecret))
{
await SetSecretAsync(Guid.NewGuid().ToString(), environment.AppEndpoint).ConfigureAwait(false);
}
if (!string.IsNullOrEmpty(environment.PartnerCenterEndpoint?.ApplicationSecret))
{
await SetSecretAsync(Guid.NewGuid().ToString(), environment.PartnerCenterEndpoint).ConfigureAwait(false);
}
await repository.AddOrUpdateAsync(environment).ConfigureAwait(false);
return(RedirectToAction("Index"));
}
return(View(environment));
}
public static async Task ProcessCustomerAsync(
[QueueTrigger(OperationConstants.CustomersQueueName, Connection = "StorageConnectionString")] XEvent data,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(CustomerDetail),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <CustomerDetail> customerRepository,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(SubscriptionDetail),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <SubscriptionDetail> subscriptionRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Resource = "https://graph.windows.net")] IPartnerServiceClient partner,
[StorageService(
ConnectionStringName = "StorageConnectionString",
KeyVaultEndpoint = "KeyVaultEndpoint")] IStorageService storage,
TraceWriter log)
{
List <SubscriptionDetail> subscriptions;
try
{
log.Info($"Processing data for {data.Customer.Id}");
if (data.Customer.LastProcessed == null || (DateTimeOffset.UtcNow - data.Customer.LastProcessed).Value.TotalDays >= 30)
{
subscriptions = await GetSubscriptionsAsync(partner, data.Customer.Id).ConfigureAwait(false);
}
else
{
subscriptions = await BuildUsingAuditRecordsAsync(
data.AuditRecords,
subscriptionRepository,
partner,
data.Customer).ConfigureAwait(false);
}
if (subscriptions.Count > 0)
{
await subscriptionRepository.AddOrUpdateAsync(subscriptions).ConfigureAwait(false);
}
data.Customer.LastProcessed = DateTimeOffset.UtcNow;
data.Customer.ProcessException = null;
await customerRepository.AddOrUpdateAsync(data.Customer).ConfigureAwait(false);
await storage.WriteToQueueAsync(
OperationConstants.SecurityQueueName,
new SecurityDetails
{
AppEndpoint = data.AppEndpoint,
Customer = data.Customer
}).ConfigureAwait(false);
log.Info($"Successfully process data for {data.Customer.Id}");
}
finally
{
subscriptions = null;
}
}
public static async Task ProcessPartnerAsync(
[QueueTrigger(OperationConstants.PartnersQueueName, Connection = "StorageConnectionString")] EnvironmentDetail environment,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(AuditRecord),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <AuditRecord> auditRecordRepository,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(CustomerDetail),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <CustomerDetail> customerRepository,
[DataRepository(
CosmosDbEndpoint = "CosmosDbEndpoint",
DataType = typeof(EnvironmentDetail),
KeyVaultEndpoint = "KeyVaultEndpoint")] IDocumentRepository <EnvironmentDetail> environmentRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
KeyVaultEndpoint = "KeyVaultEndpoint",
Resource = "https://graph.windows.net")] IPartnerServiceClient partner,
[StorageService(
ConnectionStringName = "StorageConnectionString",
KeyVaultEndpoint = "KeyVaultEndpoint")] IStorageService storage,
TraceWriter log)
{
List <AuditRecord> auditRecords;
List <CustomerDetail> customers;
SeekBasedResourceCollection <AuditRecord> seekAuditRecords;
try
{
log.Info($"Starting to process the {environment.FriendlyName} CSP environment.");
// Request the audit records for the previous day from the Partner Center API.
seekAuditRecords = await partner.AuditRecords.QueryAsync(DateTime.Now.AddDays(-1)).ConfigureAwait(false);
auditRecords = new List <AuditRecord>(seekAuditRecords.Items);
while (seekAuditRecords.Links.Next != null)
{
// Request the next page of audit records from the Partner Center API.
seekAuditRecords = await partner.AuditRecords.QueryAsync(seekAuditRecords.Links.Next).ConfigureAwait(false);
auditRecords.AddRange(seekAuditRecords.Items);
}
if (auditRecords.Count > 0)
{
// Add, or update, each audit record to the database.
await auditRecordRepository.AddOrUpdateAsync(auditRecords).ConfigureAwait(false);
}
if ((DateTimeOffset.UtcNow - environment?.LastProcessed).Value.TotalDays >= 30)
{
customers = await GetCustomersAsync(partner).ConfigureAwait(false);
}
else
{
customers = await BuildUsingAuditRecordsAsync(
auditRecords,
customerRepository).ConfigureAwait(false);
}
// Add, or update, each customer to the database.
await customerRepository.AddOrUpdateAsync(customers).ConfigureAwait(false);
foreach (CustomerDetail customer in customers)
{
// Write the customer to the customers queue to start processing the customer.
await storage.WriteToQueueAsync(
OperationConstants.CustomersQueueName,
new XEvent
{
AppEndpoint = environment.AppEndpoint,
AuditRecords = auditRecords
.Where(r => r.CustomerId.Equals(customer.Id, StringComparison.InvariantCultureIgnoreCase))
.ToList(),
Customer = customer,
PartnerCenterEndpoint = environment.PartnerCenterEndpoint
}).ConfigureAwait(false);
}
environment.LastProcessed = DateTimeOffset.UtcNow;
await environmentRepository.AddOrUpdateAsync(environment).ConfigureAwait(false);
log.Info($"Successfully process the {environment.FriendlyName} CSP environment.");
}
finally
{
auditRecords = null;
customers = null;
seekAuditRecords = null;
}
}
public static async Task ProcessCustomerAsync(
[QueueTrigger(OperationConstants.CustomersQueueName, Connection = "StorageConnectionString")] ProcessCustomerDetail customerDetail,
[DataRepository(DataType = typeof(AuditRecord))] IDocumentRepository <AuditRecord> auditRecordRepository,
[DataRepository(DataType = typeof(CustomerDetail))] IDocumentRepository <CustomerDetail> customerRepository,
[DataRepository(DataType = typeof(SubscriptionDetail))] IDocumentRepository <SubscriptionDetail> subscriptionRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartnerServiceClient partner,
[StorageService] IStorageService storage,
TraceWriter log)
{
IEnumerable <AuditRecord> auditRecords;
List <SubscriptionDetail> subscriptions;
int period;
try
{
log.Info($"Attempting to process information for {customerDetail.Customer.Id}.");
if (customerDetail.Customer.RemovedFromPartnerCenter)
{
// The customer no longer has relationship with the partner. So, it should not be processed.
return;
}
// Configure the value indicating the number of days of score results to retrieve starting from current date.
period = (customerDetail.Customer.LastProcessed == null) ? 30 : (DateTimeOffset.UtcNow - customerDetail.Customer.LastProcessed).Value.Days;
// Ensure that the period is at least 1 or greater. This ensure the request to retrieve data is succesfully.
if (period < 1)
{
period = 1;
}
if (period >= 30)
{
period = 30;
try
{
customerDetail.Customer.ProcessException = null;
subscriptions = await GetSubscriptionsAsync(
partner,
customerDetail.Customer.Id).ConfigureAwait(false);
}
catch (ServiceClientException ex)
{
customerDetail.Customer.ProcessException = ex;
subscriptions = null;
log.Warning($"Encountered an exception when processing {customerDetail.Customer.Id}. Check the customer record for more information.");
}
}
else
{
// Obtain a list of audit records for the specified customer that happened since the customer was last processed.
auditRecords = await auditRecordRepository.GetAsync(
r => r.CustomerId == customerDetail.Customer.Id &&
r.OperationDate >= customerDetail.Customer.LastProcessed,
customerDetail.Customer.EnvironmentId).ConfigureAwait(false);
subscriptions = await subscriptionRepository
.GetAsync(s => s.TenantId == customerDetail.Customer.Id, customerDetail.Customer.Id)
.ConfigureAwait(false);
// Since the period is less than 30 we can utilize the audit logs to reconstruct any subscriptions that were created.
subscriptions = await AuditRecordConverter.ConvertAsync(
partner,
auditRecords,
subscriptions,
customerDetail.Customer).ConfigureAwait(false);
}
if (subscriptions?.Count > 0)
{
await subscriptionRepository.AddOrUpdateAsync(
subscriptions,
customerDetail.Customer.Id).ConfigureAwait(false);
}
if (customerDetail.Customer.ProcessException == null)
{
await storage.WriteToQueueAsync(
OperationConstants.SecurityQueueName,
new SecurityDetail
{
AppEndpoint = customerDetail.AppEndpoint,
Customer = customerDetail.Customer,
Period = period.ToString(CultureInfo.InvariantCulture)
}).ConfigureAwait(false);
if (customerDetail.ProcessAzureUsage)
{
foreach (SubscriptionDetail subscription in subscriptions.Where(s => s.BillingType == BillingType.Usage))
{
await storage.WriteToQueueAsync(
OperationConstants.UtilizationQueueName,
new ProcessSubscriptionDetail
{
PartnerCenterEndpoint = customerDetail.PartnerCenterEndpoint,
Subscription = subscription
}).ConfigureAwait(false);
}
}
customerDetail.Customer.LastProcessed = DateTimeOffset.UtcNow;
}
await customerRepository.AddOrUpdateAsync(customerDetail.Customer).ConfigureAwait(false);
log.Info($"Successfully processed customer {customerDetail.Customer.Id}. Exception(s): {(customerDetail.Customer.ProcessException != null ? "yes" : "no")}");
}
finally
{
auditRecords = null;
subscriptions = null;
}
}
public static async Task ProcessUsageAsync(
[QueueTrigger(OperationConstants.UtilizationQueueName, Connection = "StorageConnectionString")] ProcessSubscriptionDetail subscriptionDetail,
[DataRepository(
DataType = typeof(UtilizationDetail))] IDocumentRepository <UtilizationDetail> repository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartnerServiceClient client,
TraceWriter log
)
{
// Subscriptions with a billing type of usage are the only ones that have utilization records.
if (subscriptionDetail.Subscription.BillingType != BillingType.Usage)
{
return;
}
log.Info($"Requesting utilization records for {subscriptionDetail.Subscription.Id}");
List <UtilizationDetail> records;
ResourceCollection <AzureUtilizationRecord> utilizationRecords;
try
{
utilizationRecords = await client
.Customers[subscriptionDetail.Subscription.TenantId]
.Subscriptions[subscriptionDetail.Subscription.Id]
.Utilization
.Azure
.QueryAsync(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow).ConfigureAwait(false);
records = new List <UtilizationDetail>();
if (utilizationRecords.TotalCount > 0)
{
records.AddRange(utilizationRecords.Items
.Select(r => ResourceConverter.Convert <AzureUtilizationRecord, UtilizationDetail>(
r,
new Dictionary <string, string>
{
{ "Id", $"{r.Resource.Id}--{r.UsageStartTime}" },
{ "SubscriptionId", subscriptionDetail.Subscription.Id },
{ "TenantId", subscriptionDetail.Subscription.TenantId }
})));
while (utilizationRecords.Links.Next != null)
{
utilizationRecords = await client
.Customers[subscriptionDetail.Subscription.TenantId]
.Subscriptions[subscriptionDetail.Subscription.Id]
.Utilization
.Azure
.QueryAsync(utilizationRecords.Links.Next).ConfigureAwait(false);
records.AddRange(utilizationRecords.Items
.Select(r => ResourceConverter.Convert <AzureUtilizationRecord, UtilizationDetail>(
r,
new Dictionary <string, string>
{
{ "Id", $"{r.Resource.Id}--{r.UsageStartTime}" },
{ "SubscriptionId", subscriptionDetail.Subscription.Id },
{ "TenantId", subscriptionDetail.Subscription.TenantId }
})));
}
}
if (records.Count > 0)
{
log.Info($"Writing {records.Count} utilization records to the repository.");
await repository.AddOrUpdateAsync(
records,
subscriptionDetail.Subscription.Id).ConfigureAwait(false);
}
}
finally
{
records = null;
utilizationRecords = null;
}
}
public static async Task ProcessPartnerAsync(
[QueueTrigger(OperationConstants.PartnersQueueName, Connection = "StorageConnectionString")] EnvironmentDetail environment,
[DataRepository(DataType = typeof(AuditRecord))] IDocumentRepository <AuditRecord> auditRecordRepository,
[DataRepository(DataType = typeof(CustomerDetail))] IDocumentRepository <CustomerDetail> customerRepository,
[DataRepository(
DataType = typeof(EnvironmentDetail))] IDocumentRepository <EnvironmentDetail> environmentRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartnerServiceClient client,
[StorageService] IStorageService storage,
TraceWriter log)
{
List <AuditRecord> auditRecords;
List <CustomerDetail> customers;
int days;
try
{
log.Info($"Starting to process the {environment.FriendlyName} CSP environment.");
// Calculate the number of days that have gone by, since the last successful synchronization.
days = (environment.LastProcessed == null) ? 30 : (DateTimeOffset.UtcNow - environment.LastProcessed).Days;
if (days >= 90)
{
// Only audit records for the past 90 days are available from Partner Center.
days = 89;
}
else if (days <= 0)
{
// Ensure that in all circumstances at least one day of records will be returned.
days = 1;
}
if (days >= 30)
{
customers = await GetCustomersAsync(client, environment).ConfigureAwait(false);
}
else
{
auditRecords = await GetAuditRecordsAsyc(
client,
DateTime.UtcNow.AddDays(-days),
DateTime.UtcNow).ConfigureAwait(false);
if (auditRecords.Count > 0)
{
log.Info($"Importing {auditRecords.Count} audit records available between now and the previous day.");
// Add, or update, each audit record to the database.
await auditRecordRepository.AddOrUpdateAsync(
auditRecords,
environment.Id).ConfigureAwait(false);
}
customers = await customerRepository.GetAsync().ConfigureAwait(false);
customers = await AuditRecordConverter.ConvertAsync(
client,
auditRecords,
customers,
new Dictionary <string, string> {
{ "EnvironmentId", environment.Id }
}).ConfigureAwait(false);
}
// Add, or update, each customer to the database.
await customerRepository.AddOrUpdateAsync(customers).ConfigureAwait(false);
foreach (CustomerDetail customer in customers)
{
// Write the customer to the customers queue to start processing the customer.
await storage.WriteToQueueAsync(
OperationConstants.CustomersQueueName,
new ProcessCustomerDetail
{
AppEndpoint = environment.AppEndpoint,
Customer = customer,
PartnerCenterEndpoint = environment.PartnerCenterEndpoint,
ProcessAzureUsage = environment.ProcessAzureUsage
}).ConfigureAwait(false);
}
environment.LastProcessed = DateTimeOffset.UtcNow;
await environmentRepository.AddOrUpdateAsync(environment).ConfigureAwait(false);
log.Info($"Successfully process the {environment.FriendlyName} CSP environment.");
}
finally
{
auditRecords = null;
customers = null;
}
}
