public static async Task RunAsync([BlobTrigger("%amsBlobInputContainer%/{name}.mp4", Connection =
"AzureWebJobsStorage")] CloudBlockBlob inputVideoBlob, // video blob that initiated this function
[Blob("%amsBlobInputContainer%/{name}.json", FileAccess.Read)] string manifestContents, // if a json file with the same name exists, it's content will be in this variable.
[Queue("ams-input")] IAsyncCollector <string> outputQueue, // output queue for async processing and resiliency
TraceWriter log)
{
//================================================================================
// Function AMSInputBlobWatcher
// Purpose:
// This function monitors a blob container for new mp4 video files (TODO:// update
// filter to include all video formats supported by MES). If the video files are
// accompanied by a json file with the same file name, it will use this json file
// for metadata such as video title, external ids, etc. Any custom fields added
// to this meta data file will be stored with the resulting document in Cosmos.
// ** Rather than doing any real processing here, just forward the payload to a
// queue to be more resilient. A client app can either post files to the storage
// container or add items to the queue directly. Aspera or Signiant users will
// most likely opt to use the watch folder.
// ** NOTE - the json file must be dropped into the container first.
//================================================================================
// if metadata json was used, get it's values as a dictionary
var metaDataDictionary =
!string.IsNullOrEmpty(manifestContents)
? JsonConvert.DeserializeObject <Dictionary <string, string> >(manifestContents)
: new Dictionary <string, string>();
// work out the global id for this video. If internal_id was in manifest json, use that.
// Otherwise create a new one
var globalId = metaDataDictionary.ContainsKey("internal_id")
? metaDataDictionary["internal_id"]
: Guid.NewGuid().ToString();
// add values to the state variable that is stored in Cosmos to keep track
// of various stages of processing, which also allows passing values from the json manifest
// file to the final document stored in Cosmos
var state = new VippyProcessingState
{
Id = globalId,
BlobName = inputVideoBlob.Name,
StartTime = DateTime.Now,
CustomProperties = metaDataDictionary,
};
Globals.LogMessage(log, $"Video '{inputVideoBlob.Name}' landed in watch folder" + (!string.IsNullOrEmpty(manifestContents)
? " with manifest json": "without manifest file"));
await outputQueue.AddAsync(JsonConvert.SerializeObject(state));
}
/// <summary>
/// Inserts a receipt like record in the database. This record will be updated when the processing
/// is completed with success or error details
/// </summary>
/// <param name="state">metadata provided with input video (manifest type data)</param>
/// <returns></returns>
public static async Task StoreProcessingStateRecordInCosmosAsync(VippyProcessingState state)
{
var collectionName = ProcessingStateCosmosCollectionName;
var client = GetCosmosClient(collectionName);
// upsert the json as a new document
try
{
Document r =
await client.UpsertDocumentAsync(
UriFactory.CreateDocumentCollectionUri(CosmosDatabasename, collectionName), state);
}
catch (Exception e)
{
throw new ApplicationException($"Error in StoreProcessingStateRecordInCosmosAsync:/r/n{e.Message}");
}
}
public static async Task Run([QueueTrigger("%InputQueue%", Connection = "AzureWebJobsStorage")] VippyProcessingState manifest,
[Blob("%AmsBlobInputContainer%/{BlobName}", FileAccess.ReadWrite)] CloudBlockBlob videoBlobTriggered,
[Blob("%ExistingAmsBlobInputContainer%/{BlobName}", FileAccess.ReadWrite)] CloudBlockBlob videoBlobExisting,
TraceWriter log)
{
//================================================================================
// Function AMSInputQueueHandler
// Purpose:
// This is where the start of the pipeline work begins. It will submit an encoding
// job to Azure Media Services. When that job completes asyncronously, a notification
// webhook will be called by AMS which causes the next stage of the pipeline to
// continue.
//================================================================================
CloudBlockBlob videoBlob = null;
if (manifest.Origin == Enums.OriginEnum.Existing)
{
videoBlob = videoBlobExisting;
}
else if (manifest.Origin == Enums.OriginEnum.Trigger)
{
videoBlob = videoBlobTriggered;
}
if (videoBlob == null)
{
log.Error("There is being an error, videoblog not initialized, not marked as existing or trigger or video is null");
return;
}
var context = MediaServicesHelper.Context;
var cosmosHelper = new CosmosHelper(log);
var toDeleteContainerName = Environment.GetEnvironmentVariable("AmsBlobToDeleteContainer");
// only set the starttime if it wasn't already set in blob watcher function (that way
// it works if the job is iniaited by using this queue directly
if (manifest.StartTime == null)
{
manifest.StartTime = DateTime.Now;
}
var videofileName = videoBlob.Name;
// get a new asset from the blob, and use the file name if video title attribute wasn't passed.
IAsset newAsset;
try
{
newAsset = BlobHelper.CreateAssetFromBlob(videoBlob, videofileName, log)
.GetAwaiter().GetResult();
}
catch (Exception e)
{
throw new ApplicationException($"Error occured creating asset from Blob;/r/n{e.Message}");
}
// If an internal_id was passed in the metadata, use it within AMS (AlternateId) and Cosmos(Id - main document id) for correlation.
// if not, generate a unique id. If the same id is ever reprocessed, all stored metadata
// will be overwritten.
newAsset.AlternateId = manifest.AlternateId;
newAsset.Update();
manifest.AmsAssetId = newAsset.Id;
log.Info($"Deleting the file {videoBlob.Name} from the container ");
//move the video to the to delete folder
await BlobHelper.Move(videoBlob.Container.Name, toDeleteContainerName, videofileName, log);
//move the manifest to the to delete folder
string manifestName = videofileName.Remove(videofileName.IndexOf('.')) + ".json";
log.Info($"Deleting the file {manifestName} from the container ");
await BlobHelper.Move(videoBlob.Container.Name, toDeleteContainerName, manifestName, log);
// copy blob into new asset
// create the encoding job
var job = context.Jobs.Create("MES encode from input container - ABR streaming");
// Get a media processor reference, and pass to it the name of the
// processor to use for the specific task.
var processor = MediaServicesHelper.GetLatestMediaProcessorByName("Media Encoder Standard");
var task = job.Tasks.AddNew("encoding task",
processor,
"Content Adaptive Multiple Bitrate MP4",
TaskOptions.None
);
task.Priority = 100;
task.InputAssets.Add(newAsset);
// setup webhook notification
var keyBytes = new byte[32];
// Check for existing Notification Endpoint with the name "FunctionWebHook"
var existingEndpoint = context.NotificationEndPoints.Where(e => e.Name == "FunctionWebHook").FirstOrDefault();
INotificationEndPoint endpoint;
//if (existingEndpoint != null)
//{
// endpoint = existingEndpoint;
//}
//else
try
{
endpoint = context.NotificationEndPoints.Create("FunctionWebHook",
NotificationEndPointType.WebHook, WebHookEndpoint, keyBytes);
}
catch (Exception)
{
throw new ApplicationException(
$"The endpoing address specified - '{WebHookEndpoint}' is not valid.");
}
task.TaskNotificationSubscriptions.AddNew(NotificationJobState.FinalStatesOnly, endpoint, false);
cosmosHelper.LogMessage($"Add an output asset to contain the results of the job");
// Add an output asset to contain the results of the job.
// This output is specified as AssetCreationOptions.None, which
// means the output asset is not encrypted.
task.OutputAssets.AddNew(videofileName, AssetCreationOptions.None);
// Starts the job in AMS. AMS will notify the webhook when it completes
job.Submit();
cosmosHelper.LogMessage($"Saving on cosmos DB");
// update processing progress with id and metadata payload
await cosmosHelper.StoreProcessingStateRecordInCosmosAsync(manifest);
cosmosHelper.LogMessage($"AMS encoding job submitted for {videofileName}");
}
public static async Task Run([QueueTrigger("ams-input", Connection = "AzureWebJobsStorage")] VippyProcessingState manifest,
[Blob("%amsBlobInputContainer%/{BlobName}", FileAccess.ReadWrite)] CloudBlockBlob videoBlob,
TraceWriter log)
{
//================================================================================
// Function AMSInputQueueHandler
// Purpose:
// This is where the start of the pipeline work begins. It will submit an encoding
// job to Azure Media Services. When that job completes asyncronously, a notification
// webhook will be called by AMS which causes the next stage of the pipeline to
// continue.
//================================================================================
var context = MediaServicesHelper.Context;
// only set the starttime if it wasn't already set in blob watcher function (that way
// it works if the job is iniaited by using this queue directly
if (manifest.StartTime == null)
{
manifest.StartTime = DateTime.Now;
}
var videofileName = videoBlob.Name;
var videoTitle = manifest.videoTitle ?? videofileName;
// get a new asset from the blob, and use the file name if video title attribute wasn't passed.
IAsset newAsset;
try
{
newAsset = CopyBlobHelper.CreateAssetFromBlob(videoBlob,
videoTitle, log).GetAwaiter().GetResult();
}
catch (Exception e)
{
throw new ApplicationException($"Error occured creating asset from Blob;/r/n{e.Message}");
}
// If an internal_id was passed in the metadata, use it within AMS (AlternateId) and Cosmos(Id - main document id) for correlation.
// if not, generate a unique id. If the same id is ever reprocessed, all stored metadata
// will be overwritten.
newAsset.AlternateId = manifest.AlternateId;
newAsset.Update();
manifest.AmsAssetId = newAsset.Id;
// delete the source input from the watch folder
videoBlob.DeleteIfExists();
// copy blob into new asset
// create the encoding job
var job = context.Jobs.Create("MES encode from input container - ABR streaming");
// Get a media processor reference, and pass to it the name of the
// processor to use for the specific task.
var processor = MediaServicesHelper.GetLatestMediaProcessorByName("Media Encoder Standard");
var task = job.Tasks.AddNew("encoding task",
processor,
"Content Adaptive Multiple Bitrate MP4",
TaskOptions.None
);
task.Priority = 100;
task.InputAssets.Add(newAsset);
// setup webhook notification
//byte[] keyBytes = Convert.FromBase64String(_signingKey);
var keyBytes = new byte[32];
// Check for existing Notification Endpoint with the name "FunctionWebHook"
var existingEndpoint = context.NotificationEndPoints.Where(e => e.Name == "FunctionWebHook").FirstOrDefault();
INotificationEndPoint endpoint = null;
if (existingEndpoint != null)
{
endpoint = existingEndpoint;
}
else
{
try
{
//byte[] credential = new byte[64];
endpoint = context.NotificationEndPoints.Create("FunctionWebHook",
NotificationEndPointType.WebHook, WebHookEndpoint, keyBytes);
}
catch (Exception)
{
throw new ApplicationException(
$"The endpoing address specified - '{WebHookEndpoint}' is not valid.");
}
}
task.TaskNotificationSubscriptions.AddNew(NotificationJobState.FinalStatesOnly, endpoint, false);
// Add an output asset to contain the results of the job.
// This output is specified as AssetCreationOptions.None, which
// means the output asset is not encrypted.
task.OutputAssets.AddNew(videofileName, AssetCreationOptions.None);
// Starts the job in AMS. AMS will notify the webhook when it completes
job.Submit();
// update processing progress with id and metadata payload
await Globals.StoreProcessingStateRecordInCosmosAsync(manifest);
Globals.LogMessage(log, $"AMS encoding job submitted for {videofileName}");
}
public static async Task RunAsync(
[BlobTrigger("%videoIndxerBlobInputContainer%/{name}.mp4", Connection = "AzureWebJobsStorage")] CloudBlockBlob myBlob,
[Blob("%videoIndxerBlobInputContainer%/{name}.json", FileAccess.Read)] string manifestContents, // if a json file with the same name exists, it's content will be in this variable.
string name,
TraceWriter log
)
{
// =============================================================================================
// This function is only used to watch a blob container when you want video files to be submitted
// directly to Video Indexer, outside of Azure Media Services. Just upload a video file to
// the input directory and this function will submit the video to Video Indexer, and the results
// will be stored in Cosmos Db when processing is complete
// =============================================================================================
_log = log;
// TODO: validate file types here or add file extension filters to blob trigger
// TODO: move all this into a queue based function. Too much here for blob watcher
// blob filename
var fileName = myBlob.Name;
Globals.LogMessage(log, $"Blob named {fileName} being procesed by BlobWatcher function..");
// if metadata json was used, get it's values as a dictionary
var metaDataDictionary =
!string.IsNullOrEmpty(manifestContents)
? JsonConvert.DeserializeObject <Dictionary <string, string> >(manifestContents)
: new Dictionary <string, string>();
// add a variable to state to indicate this was initiated via VideoIndexer watch folder,
// not via the beginning of the pipeline and ams encoding.
metaDataDictionary.Add("processingStartedFrom", "VideoIndexerWatchFolder");
// work out the global id for this video. If internal_id was in manifest json, use that.
// Otherwise create a new one
var globalId = metaDataDictionary.ContainsKey("internal_id")
? metaDataDictionary["internal_id"]
: Guid.NewGuid().ToString();
// add values to the state variable that is stored in Cosmos to keep track
// of various stages of processing, which also allows passing values from the json manifest
// file to the final document stored in Cosmos
var state = new VippyProcessingState
{
AlternateId = globalId,
BlobName = fileName,
StartTime = DateTime.Now,
CustomProperties = metaDataDictionary,
};
// update processing progress with id and metadata payload
await Globals.StoreProcessingStateRecordInCosmosAsync(state);
// get a SAS url for the blob
var sasUrl = Globals.GetSasUrl(myBlob);
Globals.LogMessage(log, $"Got SAS url {sasUrl}");
// call the api to process the video in VideoIndexer
var videoIndexerUniqueId = Globals.SubmitToVideoIndexerAsync(fileName, sasUrl, globalId, log).Result;
Globals.LogMessage(log, $"VideoId {videoIndexerUniqueId} submitted to Video Indexer!");
}
