public async Task AddorUpdateTestAsync()
{
Customer customer;
Customer excepted;
IDocumentRepository <Customer> customers;
Mock <IDocumentClient> client;
try
{
client = new Mock <IDocumentClient>();
excepted = GetTestCustomer();
client.Setup(c => c.UpsertDocumentAsync(
It.IsAny <Uri>(),
It.IsAny <object>(),
It.IsAny <RequestOptions>(),
It.IsAny <bool>())).Returns(Task.FromResult(new ResourceResponse <Document>(GetDocument(excepted))));
customers = new DocumentRepository <Customer>(client.Object, DatabaseId, CustomerCollectionId);
customer = await customers.AddOrUpdateAsync(excepted).ConfigureAwait(false);
Assert.AreEqual(customer.CompanyProfile.CompanyName, excepted.CompanyProfile.CompanyName);
Assert.AreEqual(customer.CompanyProfile.Domain, excepted.CompanyProfile.Domain);
Assert.AreEqual(customer.Id, excepted.Id);
}
finally
{
client = null;
customer = null;
customers = null;
excepted = null;
}
}
public static async Task ProcessSecurityAsync(
[QueueTrigger(OperationConstants.SecurityQueueName, Connection = "StorageConnectionString")] SecurityDetail data,
[DataRepository(
DataType = typeof(Graph.Alert))] DocumentRepository <Graph.Alert> securityAlertRepository,
[DataRepository(DataType = typeof(Graph.SecureScore))] DocumentRepository <Graph.SecureScore> secureScoreRepository,
[SecureScore(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
Period = "{Period}",
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <Graph.SecureScore> scores,
[SecurityAlerts(
ApplicationId = "{AppEndpoint.ApplicationId}",
CustomerId = "{Customer.Id}",
Resource = "{AppEndpoint.ServiceAddress}",
SecretName = "{AppEndpoint.ApplicationSecretId}")] List <Graph.Alert> alerts,
ILogger log)
{
if (data.Customer.ProcessException != null)
{
log.LogWarning($"Unable to process {data.Customer.Id} please check the customer's last exception for more information.");
return;
}
if (scores?.Count > 0)
{
log.LogInformation($"Importing {scores.Count} Secure Score entries from the past {data.Period} periods for {data.Customer.Id}");
await secureScoreRepository.AddOrUpdateAsync(
scores,
data.Customer.Id).ConfigureAwait(false);
}
if (alerts?.Count > 0)
{
log.LogInformation($"Importing {alerts.Count} security alert entries for {data.Customer.Id}");
await securityAlertRepository.AddOrUpdateAsync(
alerts,
data.Customer.Id).ConfigureAwait(false);
}
log.LogInformation($"Successfully process data for {data.Customer.Id}");
}
public static async Task ImportControlsAsync(
[TimerTrigger("0 0 10 * * *")] TimerInfo timerInfo,
[DataRepository(
DataType = typeof(ControlListEntry))] DocumentRepository <ControlListEntry> repository,
ILogger log)
{
CsvReader reader = null;
List <ControlListEntry> entries;
try
{
if (timerInfo.IsPastDue)
{
log.LogInformation("Execution of the function is starting behind schedule.");
}
using (StreamReader streamReader = new StreamReader(
Assembly.GetExecutingAssembly().GetManifestResourceStream(ControlListEmbeddedResource)))
{
reader = new CsvReader(streamReader);
reader.Configuration.RegisterClassMap <ControlListEntryClassMap>();
entries = reader.GetRecords <ControlListEntry>().ToList();
log.LogInformation($"Importing {entries.Count} Office 365 Secure Score controls details...");
await repository.AddOrUpdateAsync(entries).ConfigureAwait(false);
}
log.LogInformation("Successfully import Office 365 Secure Score controls.");
}
finally
{
entries = null;
reader?.Dispose();
}
}
public static async Task ProcessCustomerAsync(
[QueueTrigger(OperationConstants.CustomersQueueName, Connection = "StorageConnectionString")] ProcessCustomerDetail customerDetail,
[DataRepository(DataType = typeof(AuditRecord))] DocumentRepository <AuditRecord> auditRecordRepository,
[DataRepository(DataType = typeof(CustomerDetail))] DocumentRepository <CustomerDetail> customerRepository,
[DataRepository(DataType = typeof(SubscriptionDetail))] DocumentRepository <SubscriptionDetail> subscriptionRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartner partner,
[Queue(OperationConstants.SecurityQueueName, Connection = "StorageConnectionString")] ICollector <SecurityDetail> securityQueue,
[Queue(OperationConstants.UtilizationQueueName, Connection = "StorageConnectionString")] ICollector <ProcessSubscriptionDetail> utilizationQueue,
ILogger log)
{
IEnumerable <AuditRecord> auditRecords;
List <SubscriptionDetail> subscriptions;
int period;
try
{
log.LogInformation($"Attempting to process information for {customerDetail.Customer.Id}.");
//pull in customer details from PartnerCenter
try
{
Customer customer = await partner.Customers[customerDetail.Customer.Id].GetAsync().ConfigureAwait(false);
customerDetail.Customer.BillingProfile = customer.BillingProfile;
if (customerDetail.Customer.RemovedFromPartnerCenter == true)
{
customerDetail.Customer.LastProcessed = null;
}
customerDetail.Customer.RemovedFromPartnerCenter = false;
}
catch (ServiceClientException ex)
{
customerDetail.Customer.ProcessException = ex;
}
if (customerDetail.Customer.RemovedFromPartnerCenter)
{
// The customer no longer has relationship with the partner. So, it should not be processed.
return;
}
// Configure the value indicating the number of days of score results to retrieve starting from current date.
period = (customerDetail.Customer.LastProcessed == null) ? 30 : (DateTimeOffset.UtcNow - customerDetail.Customer.LastProcessed).Value.Days;
// Ensure that the period is at least 1 or greater. This ensure the request to retrieve data is succesfully.
if (period < 1)
{
period = 1;
}
if (period >= 30)
{
period = 30;
try
{
customerDetail.Customer.ProcessException = null;
subscriptions = await GetSubscriptionsAsync(
partner,
customerDetail.Customer.Id).ConfigureAwait(false);
}
catch (ServiceClientException ex)
{
customerDetail.Customer.ProcessException = ex;
subscriptions = null;
log.LogWarning($"Encountered an exception when processing {customerDetail.Customer.Id}. Check the customer record for more information.");
}
}
else
{
// Obtain a list of audit records for the specified customer that happened since the customer was last processed.
auditRecords = await auditRecordRepository.GetAsync(
r => r.CustomerId == customerDetail.Customer.Id &&
r.OperationDate >= customerDetail.Customer.LastProcessed,
customerDetail.Customer.EnvironmentId).ConfigureAwait(false);
subscriptions = await subscriptionRepository
.GetAsync(s => s.TenantId == customerDetail.Customer.Id, customerDetail.Customer.Id)
.ConfigureAwait(false);
// Since the period is less than 30 we can utilize the audit logs to reconstruct any subscriptions that were created.
subscriptions = await AuditRecordConverter.ConvertAsync(
partner,
auditRecords,
subscriptions,
customerDetail.Customer).ConfigureAwait(false);
}
if (subscriptions?.Count > 0)
{
await subscriptionRepository.AddOrUpdateAsync(
subscriptions,
customerDetail.Customer.Id).ConfigureAwait(false);
}
if (customerDetail.Customer.ProcessException == null)
{
securityQueue.Add(new SecurityDetail
{
AppEndpoint = customerDetail.AppEndpoint,
Customer = customerDetail.Customer,
Period = period.ToString(CultureInfo.InvariantCulture)
});
if (customerDetail.ProcessAzureUsage)
{
foreach (SubscriptionDetail subscription in subscriptions.Where(s => s.BillingType == BillingType.Usage))
{
utilizationQueue.Add(new ProcessSubscriptionDetail
{
PartnerCenterEndpoint = customerDetail.PartnerCenterEndpoint,
Subscription = subscription
});
}
}
customerDetail.Customer.LastProcessed = DateTimeOffset.UtcNow;
}
await customerRepository.AddOrUpdateAsync(customerDetail.Customer).ConfigureAwait(false);
log.LogInformation($"Successfully processed customer {customerDetail.Customer.Id}. Exception(s): {(customerDetail.Customer.ProcessException != null ? "yes" : "no")}");
}
finally
{
auditRecords = null;
subscriptions = null;
}
}
public static async Task ProcessUsageAsync(
[QueueTrigger(OperationConstants.UtilizationQueueName, Connection = "StorageConnectionString")] ProcessSubscriptionDetail subscriptionDetail,
[DataRepository(
DataType = typeof(UtilizationDetail))] DocumentRepository <UtilizationDetail> repository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartner client,
ILogger log)
{
// Subscriptions with a billing type of usage are the only ones that have utilization records.
if (subscriptionDetail.Subscription.BillingType != BillingType.Usage)
{
return;
}
log.LogInformation($"Requesting utilization records for {subscriptionDetail.Subscription.Id}");
IResourceCollectionEnumerator <ResourceCollection <AzureUtilizationRecord> > enumerator;
List <UtilizationDetail> records;
ResourceCollection <AzureUtilizationRecord> utilizationRecords;
try
{
utilizationRecords = await client
.Customers[subscriptionDetail.Subscription.TenantId]
.Subscriptions[subscriptionDetail.Subscription.Id]
.Utilization
.Azure
.QueryAsync(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow).ConfigureAwait(false);
enumerator = client.Enumerators.Utilization.Azure.Create(utilizationRecords);
records = new List <UtilizationDetail>();
while (enumerator.HasValue)
{
records.AddRange(enumerator.Current.Items
.Select(r => ResourceConverter.Convert <AzureUtilizationRecord, UtilizationDetail>(
r,
new Dictionary <string, string>
{
{ "Id", $"{r.Resource.Id}--{r.UsageStartTime}" },
{ "SubscriptionId", subscriptionDetail.Subscription.Id },
{ "TenantId", subscriptionDetail.Subscription.TenantId }
})));
await enumerator.NextAsync().ConfigureAwait(false);
}
if (records.Count > 0)
{
log.LogInformation($"Writing {records.Count} utilization records to the repository.");
await repository.AddOrUpdateAsync(
records,
subscriptionDetail.Subscription.Id).ConfigureAwait(false);
}
}
finally
{
records = null;
utilizationRecords = null;
}
}
public static async Task ProcessPartnerAsync(
[QueueTrigger(OperationConstants.PartnersQueueName, Connection = "StorageConnectionString")] EnvironmentDetail environment,
[DataRepository(DataType = typeof(AuditRecord))] DocumentRepository <AuditRecord> auditRecordRepository,
[DataRepository(DataType = typeof(CustomerDetail))] DocumentRepository <CustomerDetail> customerRepository,
[DataRepository(
DataType = typeof(EnvironmentDetail))] DocumentRepository <EnvironmentDetail> environmentRepository,
[PartnerService(
ApplicationId = "{PartnerCenterEndpoint.ApplicationId}",
Endpoint = "{PartnerCenterEndpoint.ServiceAddress}",
SecretName = "{PartnerCenterEndpoint.ApplicationSecretId}",
ApplicationTenantId = "{PartnerCenterEndpoint.TenantId}",
Resource = "https://graph.windows.net")] IPartner client,
[Queue(OperationConstants.CustomersQueueName, Connection = "StorageConnectionString")] ICollector <ProcessCustomerDetail> customerQueue,
ILogger log)
{
List <AuditRecord> auditRecords;
List <CustomerDetail> customers;
int days;
try
{
log.LogInformation($"Starting to process the {environment.FriendlyName} CSP environment.");
// Calculate the number of days that have gone by, since the last successful synchronization.
days = (environment.LastProcessed == null) ? 30 : (DateTimeOffset.UtcNow - environment.LastProcessed).Days;
if (days >= 90)
{
// Only audit records for the past 90 days are available from Partner Center.
days = 89;
}
else if (days <= 0)
{
// Ensure that in all circumstances at least one day of records will be returned.
days = 1;
}
if (days >= 30)
{
customers = await GetCustomersAsync(client, environment).ConfigureAwait(false);
}
else
{
auditRecords = await GetAuditRecordsAsyc(
client,
DateTime.UtcNow.AddDays(-days),
DateTime.UtcNow).ConfigureAwait(false);
if (auditRecords.Count > 0)
{
log.LogInformation($"Importing {auditRecords.Count} audit records available between now and the previous day for the {environment.FriendlyName} Region.");
// Add, or update, each audit record to the database.
await auditRecordRepository.AddOrUpdateAsync(
auditRecords,
environment.Id).ConfigureAwait(false);
}
customers = await customerRepository.GetAsync(c => c.EnvironmentId == environment.Id).ConfigureAwait(false);
log.LogInformation($"Retrieved {customers.Count()} customers from the repository for the {environment.FriendlyName} Region");
customers = await AuditRecordConverter.ConvertAsync(
client,
auditRecords,
customers,
new Dictionary <string, string> {
{ "EnvironmentId", environment.Id }
},
log).ConfigureAwait(false);
}
log.LogInformation($"Saving {customers.Count()} customers to the repository for the {environment.FriendlyName} Region");
// Add, or update, each customer to the database.
await customerRepository.AddOrUpdateAsync(customers).ConfigureAwait(false);
log.LogInformation($"Adding {customers.Count()} customers to the queue for processing for the {environment.FriendlyName} Region");
foreach (CustomerDetail customer in customers)
{
// Write the customer to the customers queue to start processing the customer.
customerQueue.Add(new ProcessCustomerDetail
{
AppEndpoint = environment.AppEndpoint,
Customer = customer,
PartnerCenterEndpoint = environment.PartnerCenterEndpoint,
ProcessAzureUsage = environment.ProcessAzureUsage
});
}
environment.LastProcessed = DateTimeOffset.UtcNow;
await environmentRepository.AddOrUpdateAsync(environment).ConfigureAwait(false);
log.LogInformation($"Successfully processed the {environment.FriendlyName} CSP environment.");
}
finally
{
auditRecords = null;
customers = null;
}
}