/// <summary>
/// Submits an item binary via the Records365 vNext Connector API.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public override async Task Submit(SubmitContext submitContext)
{
var binarySubmitContext = submitContext as BinarySubmitContext;
ValidateFields(binarySubmitContext);
// Submit via HTTP API Client that is generated with AutoRest
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
var result = await GetRetryPolicy(submitContext).ExecuteAsync(
async(ct) =>
{
// In case a stream is reused during submission retry, it might not be in 0 Position
// since the previous submission already read to the end. We should reset this value.
if (binarySubmitContext.Stream.CanSeek)
{
binarySubmitContext.Stream.Position = 0;
}
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesPostWithHttpMessagesAndStreamAsync(
binarySubmitContext.ConnectorConfigId.ToString(),
binarySubmitContext.ItemExternalId,
binarySubmitContext.ExternalId,
binarySubmitContext.FileName,
binarySubmitContext.CorrelationId.ToString(),
inputStream: binarySubmitContext.Stream,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
binarySubmitContext.CancellationToken
).ConfigureAwait(false);
var shouldContinue = true;
shouldContinue = await HandleSubmitResponse(submitContext, result, "Binary").ConfigureAwait(false);
if (shouldContinue)
{
await InvokeNext(submitContext).ConfigureAwait(false);
}
}
/// <summary>
/// Invokes the next element in the submission pipeline, if one exists.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
protected async Task InvokeNext(SubmitContext submitContext)
{
if (_next != null)
{
// Note the timing here may not be the most intuitive.
// The timing for any given element will be the sum of all processing in the remaining chain.
// Be aware of this when analysing the metrics.
using (var perfEvent = CreatePerformanceEvent(submitContext, nameof(Submit)))
{
try
{
submitContext.CancellationToken.ThrowIfCancellationRequested();
await _next.Submit(submitContext).ConfigureAwait(false);
}
catch (Exception ex)
{
perfEvent?.Exception(ex);
throw;
}
}
}
}
/// <summary>
/// Submits an aggregation to the Records365 vNext Connector API.
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public async override Task Submit(SubmitContext submitContext)
{
// Submit via HTTP API Client that is generated with AutoRest
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
Func <string, DateTime> parseDateTime = (string value) =>
{
DateTime result;
return(!string.IsNullOrEmpty(value) && DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out result) ? result : DateTime.UtcNow);
};
var aggregationModel = new AggregationSubmissionInputModel()
{
ItemTypeId = submitContext.ItemTypeId,
ExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.ExternalId)?.Value ?? "",
ParentExternalId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.ParentExternalId)?.Value ?? "",
Title = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Title)?.Value ?? "",
Author = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Author)?.Value ?? "",
SourceLastModifiedDate = parseDateTime(submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceLastModifiedDate)?.Value),
SourceCreatedDate = parseDateTime(submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceCreatedDate)?.Value),
SourceLastModifiedBy = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceLastModifiedBy)?.Value ?? "",
SourceCreatedBy = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SourceCreatedBy)?.Value ?? "",
ConnectorId = submitContext.ConnectorConfigId.ToString(),
Location = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.Location)?.Value ?? "",
MediaType = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.MediaType)?.Value ?? "Electronic",
BarcodeType = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.BarcodeType)?.Value ?? "",
BarcodeValue = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.BarcodeValue)?.Value ?? "",
RecordCategoryId = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.RecordCategoryID)?.Value ?? "",
SecurityProfileIdentifier = submitContext.CoreMetaData?.FirstOrDefault(metadata => metadata.Name == Fields.SecurityProfileIdentifier)?.Value ?? "",
SourceProperties = new List <SubmissionMetaDataModel>(),
Relationships = new List <RelationshipDataModel>()
};
if (submitContext.SourceMetaData != null)
{
aggregationModel.SourceProperties = submitContext.SourceMetaData;
}
if (submitContext.Relationships != null)
{
aggregationModel.Relationships = submitContext.Relationships;
}
var shouldContinue = true;
try
{
var result = await GetRetryPolicy(submitContext).ExecuteAsync(
async(ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiAggregationsPostWithHttpMessagesAsync(
aggregationModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
shouldContinue = await HandleSubmitResponse(submitContext, result, "Aggregation").ConfigureAwait(false);
}
catch (HttpOperationException ex)
when(ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict)
{
// submitted item already exists! Nothing to do but continue with the submission pipeline
LogVerbose(submitContext, nameof(Submit), $"Submission returned {ex.Response.StatusCode} : Aggregation already submitted.");
}
if (shouldContinue)
{
await InvokeNext(submitContext).ConfigureAwait(false);
}
}
private IPerformanceEvent CreatePerformanceEvent(SubmitContext submitContext, string methodName)
{
return(new PerformanceEvent(_next.GetType(), methodName, submitContext.LogPrefix(), Log));
}
/// <summary>
/// Logs a warning message, providing information from the SubmitContext.
/// </summary>
/// <param name="context"></param>
/// <param name="methodName"></param>
/// <param name="message"></param>
protected void LogWarning(SubmitContext context, string methodName, string message)
{
Log?.LogWarning(GetType(), methodName, $"{context.LogPrefix()} {message}");
}
/// <summary> /// Implement in a derived class to provide custom submit pipeline functionality. /// </summary> /// <param name="submitContext"></param> /// <returns></returns> public abstract Task Submit(SubmitContext submitContext);
/// <summary>
///
/// </summary>
/// <param name="submitContext"></param>
/// <returns></returns>
public override async Task Submit(SubmitContext submitContext)
{
var binarySubmitContext = submitContext as BinarySubmitContext;
ValidateFields(binarySubmitContext);
if (!CircuitProvider.IsCircuitClosed(out _))
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Deferred;
return;
}
//Create the DirectBinarySubmissionInputModel needed for the SaS token call
var binarySubmissionInputModel = new DirectBinarySubmissionInputModel(
binarySubmitContext.ConnectorConfigId.ToString(),
binarySubmitContext.ItemExternalId,
binarySubmitContext.ExternalId,
sourceLastModifiedDate: binarySubmitContext?.SourceLastModifiedDate,
fileSize: binarySubmitContext.Stream.Length,
fileName: binarySubmitContext.FileName,
fileHash: binarySubmitContext.FileHash,
mimeType: binarySubmitContext.MimeType ?? binarySubmitContext.SourceMetaData?.FirstOrDefault(metaInfo => metaInfo.Name == Fields.MimeType)?.Value,
correlationId: binarySubmitContext.CorrelationId.ToString()
);
// Get token and URL via Autorest-generated API call
var apiClient = ApiClientFactory.CreateApiClient(submitContext.ApiClientFactorySettings);
var retryPolicy = GetRetryPolicy(binarySubmitContext);
var result = await retryPolicy.ExecuteAsync(
async (ct) =>
{
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesGetSASTokenPostWithHttpMessagesAsync(
binarySubmissionInputModel: binarySubmissionInputModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
if (result.Response.StatusCode == HttpStatusCode.MethodNotAllowed)
{
// Direct binary submission is disabled, fall back to old submission method
await base.Submit(submitContext).ConfigureAwait(false);
return;
}
if (await HandleSubmitResponse(submitContext, result, "Binary").ConfigureAwait(false))
{
var response = result.Body as DirectBinarySubmissionResponseModel;
if (binarySubmitContext.Stream.CanSeek && binarySubmitContext.Stream.Length > response.MaxFileSize)
{
// We want to skip submission if the binary is too large. The CanSeek is to prevent NotSupportedException if
// we attempt to get the length of an unseekable stream.
// If we cannot seek the stream, we assume it's under the maxFileSize and allow submission.
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Skipped;
return;
}
if (!binarySubmitContext.Stream.CanSeek)
{
//TODO: Log that submission is was allowed to proceed since size could not be determined.
}
// Retrieve reference to a blob. Use the DefaultBlobFactory if the BlobFactory on the pipeline element has not been set
var blockBlob = BlobFactory != null?BlobFactory(response.Url) : DefaultBlobFactory(response.Url);
// Set Blob ContentType
blockBlob.Properties.ContentType = "application/octet-stream";
// If catch TooManyRequestsException, make it return a TooManyRequests Status
try
{
await RetryProvider.ExecuteWithRetry(
blockBlob.ServiceClient,
//Upload to blob
async() =>
{
await blockBlob.UploadFromStreamAsync(binarySubmitContext.Stream, binarySubmitContext.CancellationToken).ConfigureAwait(false);
},
GetType(),
nameof(Submit)).ConfigureAwait(false);
}
catch (TooManyRequestsException ex)
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.TooManyRequests;
submitContext.SubmitResult.WaitUntilTime = ex.WaitUntilTime;
return;
}
if (!string.IsNullOrWhiteSpace(binarySubmitContext.FileName))
{
blockBlob.Metadata[MetaDataKeys.ItemBinary_FileName] = EscapeBlobMetaDataValue(binarySubmitContext.FileName);
blockBlob.Metadata[MetaDataKeys.ItemBinary_CorrelationId] = EscapeBlobMetaDataValue(binarySubmitContext.CorrelationId.ToString());
// If catch TooManyRequestsException, make it return a TooManyRequests Status
try
{
await RetryProvider.ExecuteWithRetry(
blockBlob.ServiceClient,
async() =>
{
await blockBlob.SetMetadataAsync(binarySubmitContext.CancellationToken).ConfigureAwait(false);
},
GetType(),
nameof(Submit)).ConfigureAwait(false);
}
catch (TooManyRequestsException ex)
{
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.TooManyRequests;
submitContext.SubmitResult.WaitUntilTime = ex.WaitUntilTime;
return;
}
}
var notifyResult = await retryPolicy.ExecuteAsync(
async (ct) =>
{
//If the item corresponding to the submitted binary is not yet present, the platform will have to handle this.
var authHelper = ApiClientFactory.CreateAuthenticationHelper();
var headers = await authHelper.GetHttpRequestHeaders(submitContext.AuthenticationHelperSettings).ConfigureAwait(false);
return(await apiClient.ApiBinariesNotifyBinarySubmissionPostWithHttpMessagesAsync(
binarySubmissionInputModel: binarySubmissionInputModel,
customHeaders: headers,
cancellationToken: ct
).ConfigureAwait(false));
},
submitContext.CancellationToken
).ConfigureAwait(false);
if (notifyResult.Response.StatusCode != HttpStatusCode.OK)
{
var notificationStatusCode = "<No Status Code>";
if (result.Response != null)
{
notificationStatusCode = result.Response.StatusCode.ToString();
}
// An issue with notification occurred, so we must throw
throw new HttpOperationException(submitContext.LogPrefix() +
$"Submission returned {notificationStatusCode} : Notification of binary submission failed.");
}
}
else
{
return;
}
await InvokeNext(submitContext).ConfigureAwait(false);
}
/// <summary>
/// Handles the result of a call to a submission API.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="submitContext">The context of the submission.</param>
/// <param name="result">The result of the submit API call.</param>
/// <param name="itemTypeName">A string that identifies the item type (e.g., "Aggregation"). Used only for constructing log strings.</param>
/// <returns>True if the submit pipeline should continue as a result of successful submission, false otherwise.</returns>
protected async Task <bool> HandleSubmitResponse <T>(SubmitContext submitContext, HttpOperationResponse <T> result, string itemTypeName)
{
var shouldContinueSubmitPipeline = true;
if (result == null)
{
throw new ArgumentNullException($"{submitContext.LogPrefix()}Invalid {itemTypeName} submission! Expecting a return type of {nameof(HttpOperationResponse<T>)} but got null!");
}
if (result.Response == null)
{
throw new HttpOperationException($"{submitContext.LogPrefix()}Invalid {itemTypeName} submission! Expecting a return type of {nameof(HttpResponseMessage)} but got null!");
}
// Default the SubmitStatus to OK first in case any developer reuse the same SubmitContext
// Then all the following SubmitContext will have the previous result
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.OK;
switch (result.Response.StatusCode)
{
case System.Net.HttpStatusCode.Conflict:
// shouldContinueSubmitPipeline should still be true on Conflict.
// There may have been previous submissions of that item that failed at a later stage and need to be re-tried.
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} already submitted.");
break;
case System.Net.HttpStatusCode.OK:
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} submitted.");
break;
case System.Net.HttpStatusCode.Created:
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} submitted.");
break;
case System.Net.HttpStatusCode.Accepted:
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} accepted for submission.");
break;
case System.Net.HttpStatusCode.NoContent:
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} accepted for submission.");
break;
case System.Net.HttpStatusCode.BadRequest:
{
shouldContinueSubmitPipeline = false;
// BadRequest (400) is returned in one of three scenarios:
// - The submit was invalid (e.g. missing required field)
// - the connector was disabled
// - we're submitting a binary but binary protection is disabled for this connector
var errorResponse = result.Body as ErrorResponseModel;
if (errorResponse?.Error?.MessageCode == MessageCode.ConnectorNotEnabled)
{
// Connector was disabled
LogWarning(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because the connector was disabled.");
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.ConnectorDisabled;
}
else if (errorResponse?.Error?.MessageCode == MessageCode.ProtectionNotEnabled)
{
// Protection is disabled
LogWarning(submitContext, nameof(HandleSubmitResponse), $"Binary submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because the connector does not have protection enabled.");
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Skipped;
}
else
{
// Bad request
LogWarning(submitContext, nameof(HandleSubmitResponse),
$"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because the submit was invalid. " +
$"Error Message: {errorResponse?.Error?.Message ?? "<null>"} " +
$"MessageCode: {errorResponse?.Error?.MessageCode ?? "<null>"} ");
// On an invalid submission, throw an exception to encourage correct dead lettering.
// The rest of the pipeline won't execute.
var httpContent = "<null>";
if (result.Response != null && result.Response.Content != null)
{
try
{
httpContent = await result.Response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch { }
}
throw new HttpOperationException(submitContext.LogPrefix() +
$"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because the submit was invalid." +
$"Http Content: {httpContent}");
}
break;
}
case System.Net.HttpStatusCode.Forbidden:
shouldContinueSubmitPipeline = false;
LogWarning(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because the connector was not found.");
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.ConnectorNotFound;
break;
case System.Net.HttpStatusCode.PreconditionFailed:
shouldContinueSubmitPipeline = false;
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because a precondition failed.");
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.Deferred;
break;
case (System.Net.HttpStatusCode) 429:
shouldContinueSubmitPipeline = false;
if (result.Response.Headers.TryGetValues(waitUntilTime, out var values))
{
var timeValue = values?.FirstOrDefault();
if (!string.IsNullOrEmpty(timeValue) && DateTime.TryParse(timeValue, out var time))
{
submitContext.SubmitResult.WaitUntilTime = time.ToUniversalTime();
}
}
if (submitContext.SubmitResult.WaitUntilTime == null)
{
submitContext.SubmitResult.WaitUntilTime = DateTime.UtcNow.AddSeconds(defaultWaitTimeSeconds);
}
LogVerbose(submitContext, nameof(HandleSubmitResponse), $"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted because a part of the system is experiencing heavy load. " +
$"Try again after {submitContext.SubmitResult.WaitUntilTime}.");
submitContext.SubmitResult.SubmitStatus = SubmitResult.Status.TooManyRequests;
break;
default:
shouldContinueSubmitPipeline = false;
LogWarning(submitContext, nameof(HandleSubmitResponse),
$"Submission returned {result.Response.StatusCode} : {itemTypeName} NOT submitted.");
throw new HttpOperationException(submitContext.LogPrefix() +
$"Submission returned {result.Response.StatusCode} : NOT submitted. " +
$"Http Content: {await result.Response.Content.ReadAsStringAsync().ConfigureAwait(false)}");
}
return(shouldContinueSubmitPipeline);
}
/// <summary>
/// Gets a retry policy which can be used for communicating with Records365
/// </summary>
/// <returns></returns>
protected Policy GetRetryPolicy(SubmitContext submitContext, string methodName = nameof(Submit))
{
return(ApiClientRetryPolicy.GetPolicy(Log, GetType(), methodName, MaxRetryAttempts, submitContext.CancellationToken, submitContext.LogPrefix()));
}
