/// <inheritdoc/>
public async Task <object> DispatchEventGridEvents(List <EventGridEvent> eventGridEvents)
{
_ = eventGridEvents ?? throw new ArgumentNullException(nameof(eventGridEvents));
foreach (EventGridEvent eventGridEvent in eventGridEvents)
{
// Handle Subscription Validation Request:
if (eventGridEvent.EventType == EventTypes.EventGridSubscriptionValidationEvent)
{
_logger.LogEventObject(LogEventIds.ReceivedEventGridSubscriptionValidationEventType, new { EventTypes = eventGridEvent.EventType });
var eventData = JsonConvert.DeserializeObject <SubscriptionValidationEventData>(eventGridEvent.Data.ToString());
var responseData = new SubscriptionValidationResponse()
{
ValidationResponse = eventData.ValidationCode
};
return(new OkObjectResult(responseData));
}
try
{
// Handle other event types
_ = await HandleEventGridEvent(eventGridEvent).ConfigureAwait(false);
}
catch (Exception e)
{
var eventInfo = new { eventGridEvent.Id, eventGridEvent.EventType, eventGridEvent.DataVersion };
_logger.LogExceptionObject(LogEventIds.ExceptionHandlingEventGridEvent, e, eventInfo);
}
}
return(new OkResult());
}
/// <summary>
/// Publishes the provided event to a topic. Topic configuration will be fetched from the application settings using this object's
/// <see cref="ISettingsProvider"/>.
/// </summary>
/// <param name="eventToPublish">The event to be published.</param>
/// <returns>A boolean indicating if the operation was run successfully.</returns>
/// <exception cref="ArgumentException">If the configuration for the topic cannot be found in the settings.</exception>
public async Task <bool> PublishEventToTopic(EventGridEvent eventToPublish)
{
_ = eventToPublish ?? throw new ArgumentNullException(nameof(eventToPublish));
var eventInfo = new { eventToPublish.Id, eventToPublish.EventType, eventToPublish.DataVersion };
try
{
var topicKey = _settingsProvider.GetAppSettingsValue(Publishing.TopicOutboundKeySettingName);
_ = topicKey ?? throw new ArgumentException($"No topic key was found for topic with name {Publishing.TopicOutbound}");
var topicEndPointString = _settingsProvider.GetAppSettingsValue(Publishing.TopicOutboundEndpointSettingName);
_ = topicEndPointString ?? throw new ArgumentException($"No topic endpoint was found for topic with name {Publishing.TopicOutbound}");
Uri.TryCreate(topicEndPointString, UriKind.Absolute, out Uri topicEndPoint);
_ = topicEndPoint ?? throw new ArgumentException($"Failed to parse Uri for topic with name {Publishing.TopicOutbound}");
var eventGridClient = _eventGridClientProvider.GetEventGridClientForTopic(Publishing.TopicOutbound, topicKey);
_logger.LogEventObject(LogEventIds.AboutToAttemptPublishOfEventWithId, eventInfo);
await eventGridClient.PublishEventsAsync(topicEndPoint.Host, new List <EventGridEvent>() { eventToPublish }).ConfigureAwait(false);
_logger.LogEventObject(LogEventIds.PublishedEvent, eventInfo);
return(true);
}
catch (Exception e)
{
_logger.LogExceptionObject(LogEventIds.ExceptionPublishingEvent, e, eventInfo);
return(false);
}
}
/// <summary>
/// Creates a BlobClientSleeve (BlobBaseClient + Context). Called when one does not exist yet.
/// </summary>
/// <param name="blobUri">The target Uri.</param>
/// <param name="ctx">The context.</param>
/// <returns>
/// A BlobBaseClient object.
/// </returns>
internal StorageBlobClientSleeve CreateBlobClientSleeveForUri(Uri blobUri, StorageClientProviderContext ctx)
{
BlobBaseClient blobBaseClient;
BlobUriBuilder blobUriBuilder;
var contextCopy = new StorageClientProviderContext(ctx);
var blobClientOptions = new BlobClientOptions();
try
{
blobUriBuilder = new BlobUriBuilder(blobUri);
var clientRequestIdPolicy = new BlobClientPipelinePolicy(contextCopy);
blobClientOptions.AddPolicy(clientRequestIdPolicy, HttpPipelinePosition.PerCall);
blobBaseClient = new BlobBaseClient(blobUri, _identity, blobClientOptions);
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.BlobBaseClientProviderFailedToCreateBlobBaseClient, e, blobUri);
throw;
}
try
{
// BlobUriBuilder SDK class will just give null for Accountname if URL is malformed, so let flow upwards
_ = StringHelpers.NullIfNullOrWhiteSpace(blobUriBuilder.AccountName) ?? throw new UriFormatException($@"Malformed Azure Storage URL: {blobUri}");
var accountUri = StorageHelpers.BuildStorageAccountUri(blobUriBuilder.AccountName, buildForBlobService: true);
var newSleeve = new StorageBlobClientSleeve(blobBaseClient,
new BlobServiceClient(accountUri, _identity, blobClientOptions),
contextCopy);
return(newSleeve);
}
catch (UriFormatException uriex)
{
_log.LogExceptionObject(LogEventIds.BlobBaseClientProviderUriMissingAccountName, uriex, blobUri);
throw new ArgumentException(LogEventIds.BlobBaseClientProviderUriMissingAccountName.Name, nameof(blobUri), uriex);
}
catch (ArgumentException aex)
{
_log.LogExceptionObject(LogEventIds.BlobBaseClientProviderUriMissingAccountName, aex, blobUri);
throw;
}
}
/// <summary>
/// Creates a BlobContainerClient from a connection string. Called when one does not exist yet.
/// </summary>
/// <param name="connectionString">Connection string</param>
/// <param name="containerName">Container name</param>
/// <param name="context">current storage context</param>
/// <returns>
/// A BlobContainerClient object.
/// </returns>
internal StorageContainerClientSleeve CreateBlobContainerClientForConnectionString(string connectionString, string containerName, StorageClientProviderContext context)
{
var ctxCopy = new StorageClientProviderContext(context);
BlobContainerClient blobContainerClient;
BlobClientOptions clientOptions;
try
{
clientOptions = new BlobClientOptions();
var clientRequestIdPolicy = new BlobClientPipelinePolicy(ctxCopy);
blobContainerClient = new BlobContainerClient(connectionString, containerName, clientOptions);
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.BlobContainerClientProviderFailedToCreateBlobContainerClient, e, new { connectionString, containerName });
throw;
}
return(new StorageContainerClientSleeve(blobContainerClient,
new BlobServiceClient(connectionString, clientOptions),
ctxCopy));
}
/// <inheritdoc/>
protected override async Task <ResponseBaseDTO> DoWorkAsync(MediaServicesV2NotificationMessage eventData, string eventType)
{
_ = eventData ?? throw new ArgumentNullException(nameof(eventData));
var jobId = eventData.Properties["jobId"];
if (string.IsNullOrWhiteSpace(jobId))
{
throw new ArgumentException("notificationMessage.jobId is invalid");
}
// var taskId = message.Properties["TaskId"]; // TODO: this variable does not seem to be used around here
var operationContext = await _amsV2Service.GetOperationContextForJobAsync(jobId).ConfigureAwait(false);
try
{
switch (eventData.EventType)
{
case MediaServicesV2NotificationEventType.TaskStateChange:
return(await ProcessTaskStateChangeInternalAsync(eventData, jobId, operationContext).ConfigureAwait(false));
case MediaServicesV2NotificationEventType.TaskProgress:
return(GetEncodeProcessingDTO(eventData, jobId, operationContext));
default:
// TODO: what happens if the message type is not handled?
_log.LogEventObject(LogEventIds.MediaServicesV2UnableToHandleMessageEventType, eventData);
throw new NotSupportedException($"Unsupported AMSv2 event type {eventData.EventType}");
}
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.MediaServicesV2MessageProcessingFailed, e, eventData);
// TODO: ResponseEncodeFailureDTO has no data about the exception!
return(new ResponseEncodeFailureDTO
{
OperationContext = operationContext,
EncoderContext = JObject.FromObject(eventData.Properties),
WorkflowJobName = jobId
});
}
}
/// <inheritdoc/>
public async Task <JObject> GetMediaInfoCompleteInformForUriAsync(Uri blobUri, StorageClientProviderContext context)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
_ = context ?? throw new ArgumentNullException(nameof(context));
// Get MediaInfo instance
var mediaInfoLib = _mediaInfoProvider.GetMediaInfoLib(context);
// Get contentLength
var blobProperties = await _storageService.GetBlobPropertiesAsync(blobUri, context).ConfigureAwait(false);
var contentLength = blobProperties.ContentLength;
if (contentLength == 0)
{
_logger.LogEventObject(LogEventIds.InvalidBlobContentLength, blobUri);
throw new GridwichMediaInfoLibException($"Content length 0 for blob uri {blobUri}", LogEventIds.InvalidBlobContentLength, context.ClientRequestIdAsJObject);
}
long desiredOffset = 0;
do
{
// Get content for current desiredOffset
var cachedHttpRangeContent = await _storageService.GetOrDownloadContentAsync(blobUri, desiredOffset, IStorageService.UseDefaultLength, context).ConfigureAwait(false);
if (cachedHttpRangeContent == null)
{
_logger.LogEventObject(LogEventIds.MediaInfoInvalidContent, new { blobUri, desiredOffset });
throw new GridwichMediaInfoInvalidContentException(blobUri, desiredOffset, "The HTTP range obtained is invalid",
context.ClientRequestIdAsJObject);
}
byte[] mediaInfoBuffer;
try
{
// Copy to local buffer.
mediaInfoBuffer = cachedHttpRangeContent.CachedMemoryStream.ToArray();
// Tell MediaInfo what offset this represents in the file:
mediaInfoLib.OpenBufferInit(contentLength, cachedHttpRangeContent.CachedHttpRange.Offset);
}
catch (Exception e)
{
_logger.LogExceptionObject(LogEventIds.MediaInfoLibOpenBufferInitFailed, e, new { blobUri, httpRange = cachedHttpRangeContent.CachedHttpRange });
throw new GridwichMediaInfoLibUnexpectedException(blobUri, "MediaInfoLib threw an unexpected exception on buffer initialization",
LogEventIds.MediaInfoLibOpenBufferInitFailed, context.ClientRequestIdAsJObject, e);
}
// Pin and send the buffer to MediaInfo, who will read and parse it:
MediaInfoStatus result;
try
{
GCHandle gcHandle = GCHandle.Alloc(mediaInfoBuffer, GCHandleType.Pinned);
IntPtr addrOfBuffer = gcHandle.AddrOfPinnedObject();
result = (MediaInfoStatus)mediaInfoLib.OpenBufferContinue(addrOfBuffer, (IntPtr)mediaInfoBuffer.Length);
gcHandle.Free();
}
catch (Exception e) when(
e is ArgumentException ||
e is InvalidOperationException)
{
_logger.LogExceptionObject(LogEventIds.MediaInfoLibOpenBufferContinueFailed, e, new { blobUri, httpRange = cachedHttpRangeContent.CachedHttpRange });
throw new GridwichMediaInfoLibUnexpectedException(blobUri, "MediaInfoLib threw an unexpected exception on open buffer continuation",
LogEventIds.MediaInfoLibOpenBufferContinueFailed, context.ClientRequestIdAsJObject, e);
}
// Check if MediaInfo is done.
if ((result & MediaInfoStatus.Finalized) == MediaInfoStatus.Finalized)
{
_logger.LogEventObject(LogEventIds.MediaInfoFileReadFinalized, new { blobUri, httpRange = cachedHttpRangeContent.CachedHttpRange });
break;
}
try
{
// Test if MediaInfo requests to go elsewhere
desiredOffset = mediaInfoLib.OpenBufferContinueGoToGet();
}
catch (Exception e)
{
_logger.LogExceptionObject(LogEventIds.MediaInfoLibOpenBufferContinueGoToGetFailed, e, new { blobUri, desiredOffset, httpRange = cachedHttpRangeContent.CachedHttpRange });
throw new GridwichMediaInfoLibUnexpectedException(blobUri, "MediaInfoLib threw an unexpected exception on OpenBufferContinueGoToGet operation",
LogEventIds.MediaInfoLibOpenBufferContinueGoToGetFailed, context.ClientRequestIdAsJObject, e);
}
if (desiredOffset == contentLength)
{
// MediaInfo requested EndOfFile:
_logger.LogEventObject(LogEventIds.MediaInfoRequestedEndOfFile, new { blobUri, desiredOffset });
break;
}
else if (desiredOffset == MediaInfoReadForward)
{
// MediaInfo wants to continue reading forward
// Adjust the byte-range request offset
desiredOffset = (long)(cachedHttpRangeContent.CachedHttpRange.Offset + cachedHttpRangeContent.CachedHttpRange.Length);
_logger.LogEventObject(LogEventIds.MediaInfoReadNewRangeRequested, new { blobUri, desiredOffset });
}
else
{
// Specific seek requested
_logger.LogEventObject(LogEventIds.MediaInfoSeekRequested, new { blobUri, desiredOffset });
}
if (desiredOffset >= contentLength)
{
_logger.LogEventObject(LogEventIds.MediaInfoMismatchInDesiredOffset, new { blobUri, desiredOffset });
break;
}
}while (true);
// This is the end of the stream, MediaInfo must finish some work
mediaInfoLib.OpenBufferFinalize();
// Use MediaInfoLib as needed
mediaInfoLib.GetOption(CompleteOptionString, CompleteOptionValueString);
mediaInfoLib.GetOption(OutputOptionString, JsonOutputValueString);
var report = mediaInfoLib.GetInform();
if (string.IsNullOrEmpty(report))
{
_logger.LogEventObject(LogEventIds.InvalidMediaInfoLibReport, blobUri);
throw new GridwichMediaInfoLibException("MediaInfoLib was null.", LogEventIds.InvalidMediaInfoLibReport, context.ClientRequestIdAsJObject);
}
return(JsonHelpers.JsonToJObject(report, true)); // return "as-is" with MediaInfoLib member casing.
}
/// <inheritdoc/>
public async Task <string> CopyFilesIntoNewAsset(IEnumerable <Uri> filesToCopy)
{
_ = filesToCopy ?? throw new ArgumentNullException(nameof(filesToCopy));
_ = !filesToCopy.Any() ? throw new ArgumentOutOfRangeException(nameof(filesToCopy), "Count is zero") : 0;
string newAssetId;
var assetUriBuilder = new BlobUriBuilder(filesToCopy.First());
string assetName = GetInputAssetName(assetUriBuilder);
string assetAccountName = assetUriBuilder.AccountName;
Uri assetUri;
try
{
(newAssetId, assetUri) = await _mediaServicesV2RestWrapper.CreateEmptyAssetAsync(assetName, assetAccountName).ConfigureAwait(false);
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.MediaServicesV2InputAssetError, e, assetUriBuilder.ToUri());
throw new GridwichEncodeCreateJobException($"Failed to create asset for {assetUriBuilder.ToUri()}", null, e, LogEventIds.MediaServicesV2InputAssetError);
}
_log.LogEventObject(LogEventIds.MediaServicesV2AssetCreated, new { newAssetId, assetName });
try
{
// Create a new muted context for these copy operations
var internalCorrelator = new JObject();
string id = (!string.IsNullOrWhiteSpace(newAssetId)) ? newAssetId : $"G:{Guid.NewGuid()}";
internalCorrelator.Add("~AMS-V2-Encode", id);
var context = new StorageClientProviderContext(internalCorrelator, muted: true);
foreach (var fileToCopy in filesToCopy)
{
var sourceUriBuilder = new BlobUriBuilder(fileToCopy);
var destUriBuilder = new BlobUriBuilder(assetUri)
{
// we need to remove subfolders if any, as AMS v2 does not support subfolder(s) in an asset container
BlobName = sourceUriBuilder.BlobName.Split('/').Last(),
};
var exists = await _storageService.GetBlobExistsAsync(fileToCopy, context).ConfigureAwait(false);
if (!exists)
{
_log.LogEventObject(LogEventIds.MediaServicesV2AttemptToUseNonexistentBlobAsInput, fileToCopy);
throw new GridwichMediaServicesV2Exception($"Attempted to use nonexistent blob: {fileToCopy} as input to encoding.", LogEventIds.MediaServicesV2AttemptToUseNonexistentBlobAsInput, context.ClientRequestIdAsJObject);
}
var s = new Stopwatch();
s.Start();
var copyFromUriOperation = await _storageService.BlobCopy(fileToCopy, destUriBuilder.ToUri(), context).ConfigureAwait(false);
var response = await copyFromUriOperation.WaitForCompletionAsync().ConfigureAwait(false);
s.Stop();
_log.LogEventObject(LogEventIds.MediaServicesV2CopyFileCompleted, new { CopyElapsedMilliseconds = s.ElapsedMilliseconds.ToString("G", CultureInfo.InvariantCulture) });
}
await _mediaServicesV2RestWrapper.CreateFileInfosAsync(newAssetId).ConfigureAwait(false);
_log.LogEventObject(LogEventIds.MediaServicesV2CopyFileAndUpdateAssetSuccess, new { assetName, assetUri });
}
catch (Exception e) when(!(e is GridwichMediaServicesV2Exception))
{
_log.LogExceptionObject(LogEventIds.MediaServicesV2CopyFileAndUpdateAssetError, e, filesToCopy);
throw new GridwichEncodeCreateJobException($"Failed to copy {assetName} to {newAssetId}", null, e, LogEventIds.MediaServicesV2CopyFileAndUpdateAssetError);
}
return(newAssetId);
}
/// <inheritdoc/>
public string GetSasUrlForBlob(Uri blobUri, TimeSpan ttl, StorageClientProviderContext context)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
_ = context ?? throw new ArgumentNullException(nameof(context));
try
{
var blobUriBuilder = new BlobUriBuilder(blobUri);
// Create a SAS token that's valid for the TimeSpan, plus a back-off start for clock skew.
var timeRange = StorageHelpers.CreateTimeRangeForUrl(ttl);
BlobSasBuilder sasBuilder = new BlobSasBuilder
{
BlobContainerName = blobUriBuilder.BlobContainerName,
BlobName = blobUriBuilder.BlobName,
Resource = "b", // "b" is for blob
StartsOn = timeRange.StartTime,
ExpiresOn = timeRange.EndTime,
}.UnescapeTargetPath(); // Important adjustment(s) for SAS computation
sasBuilder.SetPermissions(BlobSasPermissions.Read); // read permissions only for the SAS.
var sleeve = _blobBaseClientProvider.GetBlobClientSleeveForUri(blobUri, context);
var userDelegation = sleeve.Service.GetUserDelegationKey(sasBuilder.StartsOn, sasBuilder.ExpiresOn)?.Value;
if (userDelegation == null)
{
var msg = $@"Unable to get a user delegation key from the Storage service for blob {blobUri}";
_log.LogEvent(LogEventIds.StorageServiceMissingConnectionString, msg);
throw new GridwichStorageServiceException(blobUri, msg, LogEventIds.StorageServiceMissingConnectionString, context.ClientRequestIdAsJObject);
}
var sasToken = sasBuilder.ToSasQueryParameters(userDelegation, blobUriBuilder.AccountName);
blobUriBuilder.Sas = sasToken;
// Construct the full URI, including the SAS token. AbsoluteUri (vs. ToString) is to ensure the %HH escaping is used.
return(blobUriBuilder.ToUri().AbsoluteUri);
}
catch (RequestFailedException e)
{
var msg = $@"Unable to get a user delegation key from the Storage service for blob {blobUri}";
_log.LogEvent(LogEventIds.StorageServiceMissingConnectionString, msg);
throw new GridwichStorageServiceException(blobUri, msg, LogEventIds.StorageServiceMissingConnectionString, context.ClientRequestIdAsJObject, e);
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.FailedToCreateBlobSasUriInStorageService, e, blobUri);
throw new GridwichStorageServiceException(blobUri, "Failed to generate the SAS url.",
LogEventIds.FailedToCreateBlobSasUriInStorageService, context.ClientRequestIdAsJObject, e);
}
}