/// <summary>
/// Submits an audit event via the Records365 vNext Connector API.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public override async Task Submit(SubmitContext submitContext)
{
// Submit via HTTP API Client that is generated with AutoRest
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
Func <string, DateTime> parseDateTime = (string value) =>
{
DateTime result;
return(!string.IsNullOrEmpty(value) && DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out result) ? result : DateTime.UtcNow);
};
var auditEventModel = new ConnectorAuditEventModel()
{
EventExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.EventExternalId)?.Value ?? "",
ItemExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.ExternalId)?.Value ?? "",
CreatedDate = parseDateTime(submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.Created)?.Value),
Description = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.Description)?.Value ?? "",
EventType = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.EventType)?.Value ?? "",
UserName = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.UserName)?.Value ?? "",
UserId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.AuditEvent.UserId)?.Value ?? "",
ConnectorId = submitContext.ConnectorConfigId.ToString(),
SourceProperties = new List <SubmissionMetaDataModel>(),
};
if (submitContext.SourceMetaData != null)
{
auditEventModel.SourceProperties = submitContext.SourceMetaData;
}
var shouldContinue = true;
try
{
var result = await GetRetryPolicy(submitContext).ExecuteAsync(
async(ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiAuditEventsPutWithHttpMessagesAsync(
auditEventModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
shouldContinue = await HandleSubmitResponse(submitContext, result, "AuditEvent").ConfigureAwait(false);
}
catch (HttpOperationException ex)
when(ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict)
{
// submitted item already exists! Nothing to do but continue with the submission pipeline
LogVerbose(submitContext, nameof(Submit), $"Submission returned {ex.Response.StatusCode} : AuditEvent already submitted.");
}
if (shouldContinue)
{
await InvokeNext(submitContext).ConfigureAwait(false);
}
}
private IApiClient GetApiClient(ITestConfigurationProvider testConfig)
{
if (TestConfig.GetBooleanValue("mockHttp").GetValueOrDefault(false))
{
MockHttpMessageHandler?mockHttp = new MockHttpMessageHandler();
OpenIdConfiguration?openIdConfigData = TestConfig.GetOpenBankingOpenIdConfiguration();
string?openIdConfig = JsonConvert.SerializeObject(openIdConfigData);
mockHttp.When(method: HttpMethod.Get, url: "https://issuer.com/.well-known/openid-configuration")
.Respond(mediaType: "application/json", content: openIdConfig);
mockHttp.When(method: HttpMethod.Get, url: "").Respond(mediaType: "application/json", content: "{}");
mockHttp.When(method: HttpMethod.Post, url: "").Respond(mediaType: "application/json", content: "{}");
HttpClient?client = mockHttp.ToHttpClient();
return(new ApiClient(instrumentation: Substitute.For <IInstrumentationClient>(), httpClient: client));
}
X509Certificate2?certificate = CertificateFactories.GetCertificate2FromPem(
privateKey: TestConfig.GetValue("transportcertificatekey"),
pem: TestConfig.GetValue("transportCertificate"));
HttpMessageHandler?handler = new HttpRequestBuilder()
.SetClientCertificate(certificate)
.CreateMessageHandler();
return(ApiClientFactory.CreateApiClient(handler));
}
public void CreateApiClient_WhenCalledInParallel_CreatesTheRightSingletonApiClientAndSetsSecurityProtocolCorrectly()
{
// Arrange
var sutApiClientFactory = new ApiClientFactory();
var apiClientFactorySettings = new ApiClientFactorySettings()
{
ConnectorApiUrl = DummyEndPointUrl,
ServerCertificateValidation = false
};
// Act
Func <IApiClient> func = () => sutApiClientFactory.CreateApiClient(apiClientFactorySettings);
int parallelTaskCount = 100;
Task <IApiClient>[] tasks = new Task <IApiClient> [parallelTaskCount];
for (int i = 0; i < parallelTaskCount; i++)
{
tasks[i] = Task.Factory.StartNew <IApiClient>(func);
}
Task.WaitAll(tasks);
var results = tasks.Select(t => t.GetAwaiter().GetResult()).ToList();
// Assert
results.Count.Should().Be(parallelTaskCount);
var firstResult = results.First();
firstResult.BaseUri.Should().Be(new Uri(DummyEndPointUrl), "Base URL is incorrect");
ServicePointManager.SecurityProtocol.Should().Be(SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12);
// Make sure that it's a singleton object
results.ForEach(result => result.Should().BeSameAs(firstResult));
ServicePointManager.ServerCertificateValidationCallback(null, null, null, System.Net.Security.SslPolicyErrors.None).Should().BeTrue();
}
public void CreateApiClient_WhenCalledFromASingleThread_CreatesCorrectApiClientAndSetsSecurityProtocolCorrectly()
{
// Arrange
var sutApiClientFactory = new ApiClientFactory();
var apiClientFactorySettings = new ApiClientFactorySettings()
{
ConnectorApiUrl = DummyEndPointUrl,
ServerCertificateValidation = false
};
// Act
var result = sutApiClientFactory.CreateApiClient(apiClientFactorySettings);
// Assert
result.Should().NotBeNull();
ServicePointManager.SecurityProtocol.Should().Be(SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12);
result.BaseUri.Should().Be(new Uri(DummyEndPointUrl), "Base URL is incorrect");
ServicePointManager.ServerCertificateValidationCallback(null, null, null, System.Net.Security.SslPolicyErrors.None).Should().BeTrue();
}
/// <summary>
/// Submits an item binary via the Records365 vNext Connector API.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public override async Task Submit(SubmitContext submitContext)
{
var binarySubmitContext = submitContext as BinarySubmitContext;
ValidateFields(binarySubmitContext);
// Submit via HTTP API Client that is generated with AutoRest
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
var result = await GetRetryPolicy(submitContext).ExecuteAsync(
async(ct) =>
{
// In case a stream is reused during submission retry, it might not be in 0 Position
// since the previous submission already read to the end. We should reset this value.
if (binarySubmitContext.Stream.CanSeek)
{
binarySubmitContext.Stream.Position = 0;
}
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesPostWithHttpMessagesAndStreamAsync(
binarySubmitContext.ConnectorConfigId.ToString(),
binarySubmitContext.ItemExternalId,
binarySubmitContext.ExternalId,
binarySubmitContext.FileName,
binarySubmitContext.CorrelationId.ToString(),
inputStream: binarySubmitContext.Stream,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
binarySubmitContext.CancellationToken
).ConfigureAwait(false);
var shouldContinue = true;
shouldContinue = await HandleSubmitResponse(submitContext, result, "Binary").ConfigureAwait(false);
if (shouldContinue)
{
await InvokeNext(submitContext).ConfigureAwait(false);
}
}
/// <summary>
/// Gets the ConnectorConfigModel from the Records365 Connector API.
/// </summary>
/// <param name="connectorConfigId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <ConnectorConfigModel> GetConnectorConfig(Guid connectorConfigId, CancellationToken cancellationToken)
{
var client = ApiClientFactory.CreateApiClient(ApiClientFactorySettings);
var policy = ApiClientRetryPolicy.GetPolicy(4, cancellationToken);
var connectorConfig = await policy.ExecuteAsync(
async (ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(AuthenticationHelperSettings).ConfigureAwait(false);
var response = await client.ApiConnectorConfigurationsByIdGetWithHttpMessagesAsync(
connectorConfigId,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false);
return(response.Body);
},
cancellationToken
).ConfigureAwait(false);
return(connectorConfig);
}
private static void Test_ApiClient(
string developerToken,
string applicationClientId,
string usernameLoginHint,
long customerId,
long accountId)
{
var authDataFactory = new AuthenticationTokenFactory(
applicationClientId,
usernameLoginHint,
tokenCacheCorrelationId: "default");
var clientFactory = new ApiClientFactory(developerToken, authDataFactory, null, accountId);
using (var customerManagementApi = clientFactory.CreateApiClient <ICustomerManagementService>())
{
var client = customerManagementApi.Client;
Trace.WriteLine(Environment.NewLine + "> GetUser: "******"> FindAccounts: ");
client.DebugExecute(c => c.FindAccounts(new FindAccountsRequest()
{
TopN = 10
}));
Trace.WriteLine(Environment.NewLine + "> GetAccount: ");
client.DebugExecute(c => c.GetAccount(new GetAccountRequest()
{
AccountId = accountId
}));
}
using (var customerBillingApi = clientFactory.CreateApiClient <ICustomerBillingService>())
{
var client = customerBillingApi.Client;
Trace.WriteLine(Environment.NewLine + "> GetAccountMonthlySpend: ");
client.DebugExecute(c => c.GetAccountMonthlySpend(
new GetAccountMonthlySpendRequest()
{
AccountId = accountId,
MonthYear = DateTime.UtcNow
}));
}
using (var campaignManagementApi = clientFactory.CreateApiClient <ICampaignManagementService>())
{
var client = campaignManagementApi.Client;
Trace.WriteLine(Environment.NewLine + "> GetCampaignsByAccountId: ");
client.DebugExecute(c => c.GetCampaignsByAccountId(
new GetCampaignsByAccountIdRequest()
{
//CustomerAccountId = accountId.ToString(), // set in header
AccountId = accountId,
CampaignType = CampaignType.Search,
}));
}
using (var bulkApi = clientFactory.CreateApiClient <IBulkService>())
{
var client = bulkApi.Client;
Trace.WriteLine(Environment.NewLine + "> DownloadCampaignsByAccountIds: ");
client.DebugExecute(c => c.DownloadCampaignsByAccountIds(
new DownloadCampaignsByAccountIdsRequest()
{
//CustomerAccountId = accountId.ToString(), // set in header
AccountIds = new long[] { accountId },
DataScope = DataScope.EntityData,
DownloadEntities = new DownloadEntity[] { DownloadEntity.Campaigns },
FormatVersion = "6.0"
}));
}
using (var reportingApi = clientFactory.CreateApiClient <IReportingService>())
{
var client = reportingApi.Client;
Trace.WriteLine(Environment.NewLine + "> SubmitGenerateReport: ");
var reportResponse = client.DebugExecute(c => c.SubmitGenerateReport(
new SubmitGenerateReportRequest()
{
ReportRequest = new BudgetSummaryReportRequest
{
ReportName = Guid.NewGuid().ToString(),
Format = ReportFormat.Tsv,
Scope = new AccountThroughCampaignReportScope
{
AccountIds = new long[] { accountId },
},
Time = new ReportTime
{
PredefinedTime = ReportTimePeriod.Last30Days
},
Columns = new[]
{
BudgetSummaryReportColumn.AccountId,
BudgetSummaryReportColumn.AccountName,
BudgetSummaryReportColumn.Date,
BudgetSummaryReportColumn.MonthlyBudget,
BudgetSummaryReportColumn.DailySpend
}
}
}));
if (reportResponse != null)
{
Trace.WriteLine(Environment.NewLine + "> PollGenerateReport: ");
client.DebugExecute(c => c.PollGenerateReport(
new PollGenerateReportRequest()
{
ReportRequestId = reportResponse.ReportRequestId
}));
}
}
}
/// <summary>
/// Submits an aggregation to the Records365 vNext Connector API.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public async override Task Submit(SubmitContext submitContext)
{
// Submit via HTTP API Client that is generated with AutoRest
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
Func <string, DateTime> parseDateTime = (string value) =>
{
DateTime result;
return(!string.IsNullOrEmpty(value) && DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out result) ? result : DateTime.UtcNow);
};
var aggregationModel = new AggregationSubmissionInputModel()
{
ItemTypeId = submitContext.ItemTypeId,
ExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.ExternalId)?.Value ?? "",
ParentExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.ParentExternalId)?.Value ?? "",
Title = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Title)?.Value ?? "",
Author = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Author)?.Value ?? "",
SourceLastModifiedDate = parseDateTime(submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceLastModifiedDate)?.Value),
SourceCreatedDate = parseDateTime(submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceCreatedDate)?.Value),
SourceLastModifiedBy = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceLastModifiedBy)?.Value ?? "",
SourceCreatedBy = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceCreatedBy)?.Value ?? "",
ConnectorId = submitContext.ConnectorConfigId.ToString(),
Location = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Location)?.Value ?? "",
MediaType = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.MediaType)?.Value ?? "Electronic",
BarcodeType = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.BarcodeType)?.Value ?? "",
BarcodeValue = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.BarcodeValue)?.Value ?? "",
RecordCategoryId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.RecordCategoryID)?.Value ?? "",
SecurityProfileIdentifier = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SecurityProfileIdentifier)?.Value ?? "",
SourceProperties = new List <SubmissionMetaDataModel>(),
Relationships = new List <RelationshipDataModel>()
};
if (submitContext.SourceMetaData != null)
{
aggregationModel.SourceProperties = submitContext.SourceMetaData;
}
if (submitContext.Relationships != null)
{
aggregationModel.Relationships = submitContext.Relationships;
}
var shouldContinue = true;
try
{
var result = await GetRetryPolicy(submitContext).ExecuteAsync(
async(ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiAggregationsPostWithHttpMessagesAsync(
aggregationModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
shouldContinue = await HandleSubmitResponse(submitContext, result, "Aggregation").ConfigureAwait(false);
}
catch (HttpOperationException ex)
when(ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict)
{
// submitted item already exists! Nothing to do but continue with the submission pipeline
LogVerbose(submitContext, nameof(Submit), $"Submission returned {ex.Response.StatusCode} : Aggregation already submitted.");
}
if (shouldContinue)
{
await InvokeNext(submitContext).ConfigureAwait(false);
}
}
/// <summary>
///
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public override async Task Submit(SubmitContext submitContext)
{
var binarySubmitContext = submitContext as BinarySubmitContext;
ValidateFields(binarySubmitContext);
if (!CircuitProvider.IsCircuitClosed(out _))
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Deferred;
return;
}
//Create the DirectBinarySubmissionInputModel needed for the SaS token call
var binarySubmissionInputModel = new DirectBinarySubmissionInputModel(
binarySubmitContext.ConnectorConfigId.ToString(),
binarySubmitContext.ItemExternalId,
binarySubmitContext.ExternalId,
sourceLastModifiedDate: binarySubmitContext?.SourceLastModifiedDate,
fileSize: binarySubmitContext.Stream.Length,
fileName: binarySubmitContext.FileName,
fileHash: binarySubmitContext.FileHash,
mimeType: binarySubmitContext.MimeType ?? binarySubmitContext.SourceMetaData?.FirstOrDefault(metaInfo => metaInfo.Name == Fields.MimeType)?.Value,
correlationId: binarySubmitContext.CorrelationId.ToString()
);
// Get token and URL via Autorest-generated API call
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
var retryPolicy = GetRetryPolicy(binarySubmitContext);
var result = await retryPolicy.ExecuteAsync(
async (ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesGetSASTokenPostWithHttpMessagesAsync(
binarySubmissionInputModel: binarySubmissionInputModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
if (result.Response.StatusCode == HttpStatusCode.MethodNotAllowed)
{
// Direct binary submission is disabled, fall back to old submission method
await base.Submit(submitContext).ConfigureAwait(false);
return;
}
if (await HandleSubmitResponse(submitContext, result, "Binary").ConfigureAwait(false))
{
var response = result.Body as DirectBinarySubmissionResponseModel;
if (binarySubmitContext.Stream.CanSeek && binarySubmitContext.Stream.Length > response.MaxFileSize)
{
// We want to skip submission if the binary is too large. The CanSeek is to prevent NotSupportedException if
// we attempt to get the length of an unseekable stream.
// If we cannot seek the stream, we assume it's under the maxFileSize and allow submission.
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Skipped;
return;
}
if (!binarySubmitContext.Stream.CanSeek)
{
//TODO: Log that submission is was allowed to proceed since size could not be determined.
}
// Retrieve reference to a blob. Use the DefaultBlobFactory if the BlobFactory on the pipeline element has not been set
var blockBlob = BlobFactory != null?BlobFactory(response.Url) : DefaultBlobFactory(response.Url);
// Set Blob ContentType
blockBlob.Properties.ContentType = "application/octet-stream";
// If catch TooManyRequestsException, make it return a TooManyRequests Status
try
{
await RetryProvider.ExecuteWithRetry(
blockBlob.ServiceClient,
//Upload to blob
async() =>
{
await blockBlob.UploadFromStreamAsync(binarySubmitContext.Stream, binarySubmitContext.CancellationToken).ConfigureAwait(false);
},
GetType(),
nameof(Submit)).ConfigureAwait(false);
}
catch (TooManyRequestsException ex)
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.TooManyRequests;
submitContext.SubmitResult.WaitUntilTime = ex.WaitUntilTime;
return;
}
if (!string.IsNullOrWhiteSpace(binarySubmitContext.FileName))
{
blockBlob.Metadata[MetaDataKeys.ItemBinary_FileName] = EscapeBlobMetaDataValue(binarySubmitContext.FileName);
blockBlob.Metadata[MetaDataKeys.ItemBinary_CorrelationId] = EscapeBlobMetaDataValue(binarySubmitContext.CorrelationId.ToString());
// If catch TooManyRequestsException, make it return a TooManyRequests Status
try
{
await RetryProvider.ExecuteWithRetry(
blockBlob.ServiceClient,
async() =>
{
await blockBlob.SetMetadataAsync(binarySubmitContext.CancellationToken).ConfigureAwait(false);
},
GetType(),
nameof(Submit)).ConfigureAwait(false);
}
catch (TooManyRequestsException ex)
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.TooManyRequests;
submitContext.SubmitResult.WaitUntilTime = ex.WaitUntilTime;
return;
}
}
var notifyResult = await retryPolicy.ExecuteAsync(
async (ct) =>
{
//If the item corresponding to the submitted binary is not yet present, the platform will have to handle this.
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesNotifyBinarySubmissionPostWithHttpMessagesAsync(
binarySubmissionInputModel: binarySubmissionInputModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
if (notifyResult.Response.StatusCode != HttpStatusCode.OK)
{
var notificationStatusCode = "<No Status Code>";
if (result.Response != null)
{
notificationStatusCode = result.Response.StatusCode.ToString();
}
// An issue with notification occurred, so we must throw
throw new HttpOperationException(submitContext.LogPrefix() +
$"Submission returned {notificationStatusCode} : Notification of binary submission failed.");
}
}
else
{
return;
}
await InvokeNext(submitContext).ConfigureAwait(false);
}
