private async Task <List <string> > UnZipAsset(OVEAssetModel asset, Stream zipFile)
{
string prefixFolder = asset.StorageLocation.Split("/").FirstOrDefault() + "/";
List <string> filesUploaded = new List <string>();
var archive = new ZipArchive(zipFile);
foreach (var entry in archive.Entries.Where(f => f.FullName.Contains(".")))
{
// only files
var location = UnzipLocation(entry.FullName, prefixFolder);
using (var file = entry.Open()) {
//raw unzipped streams do not have length so copy to memory stream
using (var ms = new MemoryStream()) {
await file.CopyToAsync(ms);
await _fileOps.Upload(asset.Project, location, ms);
}
}
filesUploaded.Add(location);
}
return(filesUploaded);
}
public string ResolveFileUrl(OVEAssetModel asset)
{
var url = _configuration.GetValue <string>(S3ClientServiceUrl).EnsureTrailingSlash()
+ asset.Project + "/" + asset.StorageLocation;
return(url);
}
#pragma warning disable 1998
public async Task <bool> Move(OVEAssetModel oldAsset, OVEAssetModel newAsset)
{
#pragma warning restore 1998
//todo this is hard because we might have to move between s3 buckets and that is complex
// https://stackoverflow.com/questions/9664904/best-way-to-move-files-between-s3-buckets
throw new NotImplementedException();
}
/// <summary>
/// Convert a numeric processing state into a user friendly message.
/// </summary>
/// <param name="asset">asset</param>
/// <returns>user friendly message</returns>
// ReSharper disable once UnusedMember.Global << used in view
public string TranslateProcessingState(OVEAssetModel asset)
{
var service = GetService(asset.Service);
if (service == null)
{
return("Unknown Service");
}
if (service.ProcessingStates.TryGetValue(asset.ProcessingState.ToString(), out string message))
{
return(message);
}
return(asset.ProcessingState == 0 ? "Unprocessed" : "unknown");
}
// ReSharper disable once UnusedMember.Global << used in view
public string GetViewUrl(OVEAssetModel model)
{
var service = GetService(model.Service);
if (service == null)
{
return("");
}
var url = service.ViewIFrameUrl;
return(string.IsNullOrWhiteSpace(url)
? ""
: url.Replace("{id}", model.Id));
}
public async Task <bool> Save(OVEAssetModel asset, IFormFile upload)
{
_logger.LogInformation("about to upload " + asset);
// set up the filename
var ext = SanitizeExtension(Path.GetExtension(upload.FileName).ToLower());
asset.StorageLocation = Guid.NewGuid() + "/" +
SanitizeFilename(Path.GetFileNameWithoutExtension(upload.FileName), ext) + ext;
try {
using (var s3Client = GetS3Client(_configuration)) {
// find or create the bucket
var buckets = await s3Client.ListBucketsAsync();
if (buckets.Buckets.FirstOrDefault(b => b.BucketName == asset.Project) == null)
{
var res = await s3Client.PutBucketAsync(asset.Project);
if (res.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception("could not create bucket" + asset.Project);
}
var openBucketPolicy =
"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetBucketLocation\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::" +
asset.Project +
"\"]},{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetObject\"],\"Resource\":[\"arn:aws:s3:::" +
asset.Project + "/*\"]}]}";
await s3Client.PutBucketPolicyAsync(asset.Project, openBucketPolicy);
_logger.LogInformation("Created bucket " + asset.Project);
}
// upload the asset
await Upload(s3Client, asset.Project, asset.StorageLocation, upload);
}
_logger.LogInformation("uploaded " + asset);
return(true);
}
catch (Exception e) {
_logger.LogError(e, "Failed to upload file for " + asset);
return(false);
}
}
public string DownloadAsset(string url, OVEAssetModel asset)
{
// make temp directory
// download url
string localFile = Path.Combine(GetLocalAssetBasePath(), asset.StorageLocation);
Directory.CreateDirectory(Path.GetDirectoryName(localFile));
_logger.LogInformation("About to download to " + localFile);
using (var client = new WebClient()) {
client.DownloadFile(new Uri(url), localFile);
}
_logger.LogInformation("Finished downloading to " + localFile);
return(localFile.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar));
}
public async Task <bool> Delete(OVEAssetModel asset)
{
_logger.LogInformation("about to delete file " + asset);
try {
using (var s3Client = GetS3Client(_configuration)) {
// delete folder containing the file and everything
ListObjectsResponse files = null;
while (files == null || files.S3Objects.Any())
{
if (files != null && files.S3Objects.Any())
{
foreach (var o in files.S3Objects)
{
await s3Client.DeleteObjectAsync(asset.Project, o.Key);
}
}
// find more files
files = await s3Client.ListObjectsAsync(new ListObjectsRequest {
BucketName = asset.Project,
Prefix = asset.GetStorageGuid()
});
}
// if the bucket is empty then delete it
var res = await s3Client.ListObjectsAsync(asset.Project);
if (!res.S3Objects.Any())
{
await s3Client.DeleteBucketAsync(asset.Project);
}
}
_logger.LogInformation("deleted file on s3 correctly");
return(true);
}
catch (Exception e) {
_logger.LogError(e, "Failed to delete an s3 file for " + asset);
return(false);
}
}
/// <summary>
/// This method calls the asset manger seek an asset to process
/// </summary>
/// <param name="state"></param>
private async void ProcessAsset(object state)
{
if (!_processing.Wait(10))
{
_logger.LogInformation("Tried to fire an asset processor but too many threads already running");
return;
}
_logger.LogInformation("About to seek an Asset to process");
OVEAssetModel asset = null;
try {
// 1) get an Asset to process
asset = await FindAssetToProcess();
if (asset == null)
{
_logger.LogInformation("no work for an asset Processor, running Processors = " +
(_maxConcurrent - _processing.CurrentCount - 1));
}
else
{
_logger.LogInformation("Found asset " + asset.Id);
await _processor.Process(this, asset);
}
} catch (Exception e) {
_logger.LogError(e, "Exception in Asset Processing");
try {
if (asset != null)
{
await UpdateStatus(asset, ProcessingErrorState, e.ToString());
}
} catch (Exception ex) {
_logger.LogError(ex, "Exception in Asset Processing");
}
} finally {
_processing.Release();
_logger.LogInformation("released processing lock");
}
}
private async Task <string> GetAssetUri(OVEAssetModel asset)
{
string url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
_configuration.GetValue <string>("AssetUrlApi") +
asset.Id;
using (var client = new HttpClient()) {
var responseMessage = await client.GetAsync(url);
if (responseMessage.StatusCode == HttpStatusCode.OK)
{
var assetString = await responseMessage.Content.ReadAsStringAsync();
_logger.LogInformation("About to download asset from url " + assetString);
return(assetString);
}
}
throw new Exception("Failed to get download URL for asset");
}
private async Task <bool> UpdateAssetMeta(OVEAssetModel asset)
{
var url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
_configuration.GetValue <string>("UpdateMetaApi") +
asset.Id + ".json";
_logger.LogInformation("Updating Asset Metadata");
using (var client = new HttpClient()) {
var body = new StringContent(asset.AssetMeta);
body.Headers.ContentType = MediaTypeHeaderValue.Parse("text/plain");
var responseMessage = await client.PostAsync(url, body);
if (responseMessage.StatusCode != HttpStatusCode.OK)
{
_logger.LogError("Failed to update the asset metadata " + responseMessage.StatusCode);
return(false);
}
}
return(true);
}
public async Task <ActionResult <OVEAssetModel> > Create(
[Bind("Project,Name,Description,Service,AssetMeta")] OVEAssetModel oveAssetModel,
[FromForm] IFormFile upload)
{
// check if we have a file
if (upload == null || upload.Length <= 0)
{
_logger.LogError("failed to upload a file");
ModelState.AddModelError("Filename", "Failed to upload file");
}
else if (!_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, upload))
{
ModelState.AddModelError("Service", "Service does not support File Type");
}
if (ModelState.IsValid)
{
// then try and save it
try {
await _fileOperations.Save(oveAssetModel, upload);
_logger.LogInformation("received and uploaded a file :) " + oveAssetModel.StorageLocation);
}
catch (Exception e) {
_logger.LogError(e, "failed to upload a file and write it to " + oveAssetModel.StorageLocation);
ModelState.AddModelError("StorageLocation", "Failed to upload file");
}
}
if (ModelState.IsValid)
{
_context.Add(oveAssetModel);
await _context.SaveChangesAsync();
return(RedirectToAction(nameof(Index)));
}
return(this.FormatOrView(oveAssetModel));
}
private async Task <OVEAssetModel> FindAssetToProcess()
{
OVEAssetModel todo = null;
string url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
_configuration.GetValue <string>("WorkItemApi") +
_configuration.GetValue <string>("ServiceName") + ".json";
_logger.LogInformation("about to get work item from url " + url);
using (var client = new HttpClient()) {
var responseMessage = await client.GetAsync(url);
_logger.LogInformation("Response was " + responseMessage.StatusCode);
if (responseMessage.StatusCode == HttpStatusCode.OK)
{
var assetString = await responseMessage.Content.ReadAsStringAsync();
todo = JsonConvert.DeserializeObject <OVEAssetModel>(assetString);
}
}
return(todo);
}
public async Task <bool> UpdateStatus(OVEAssetModel asset, int state, string errors = null) // todo int should be TV, see type constraint
{
var url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
_configuration.GetValue <string>("SetStateApi") +
asset.Id + "/" + state;
if (errors != null)
{
url += "?message=" + Uri.EscapeDataString(errors);
}
_logger.LogInformation("Setting Asset State to " + state);
using (var client = new HttpClient()) {
var responseMessage = await client.PostAsync(url, new StringContent(""));
if (responseMessage.StatusCode != HttpStatusCode.OK)
{
_logger.LogError("Failed to set asset status " + responseMessage.StatusCode);
return(false);
}
}
return(true);
}
public async Task Process(IAssetProcessingService <ImageProcessingStates> service, OVEAssetModel asset)
{
// 2) download it
string url = await service.GetAssetUri(asset);
string localUri = service.DownloadAsset(url, asset);
// 3) Create DZI file
await service.UpdateStatus(asset, (int)ImageProcessingStates.CreatingDZI);
var res = ProcessFile(localUri);
_logger.LogInformation("Processed file " + res);
// 4) Upload it
await service.UpdateStatus(asset, (int)ImageProcessingStates.Uploading);
await _fileOps.UploadIndexFileAndDirectory(Path.ChangeExtension(localUri, ".dzi"),
Path.ChangeExtension(localUri, ".dzi").Replace(".dzi", "_files/"),
asset);
// 5) delete local files
_logger.LogInformation("about to delete files");
Directory.Delete(Path.GetDirectoryName(localUri), true);
// 6) Mark it as completed
await service.UpdateStatus(asset, (int)ImageProcessingStates.Processed);
}
public async Task <bool> UploadIndexFileAndDirectory(string file, string directory, OVEAssetModel asset)
{
_logger.LogInformation($"about to upload index {file} and directory {directory}");
using (var fileTransferUtility = new TransferUtility(GetS3Client(_configuration))) {
// upload the index file
var assetRootFolder = Path.GetDirectoryName(asset.StorageLocation);
var filesKeyPrefix = assetRootFolder + "/" + new DirectoryInfo(directory).Name + "/"; // upload to the right folder
TransferUtilityUploadRequest req = new TransferUtilityUploadRequest {
BucketName = asset.Project,
Key = assetRootFolder + "/" + Path.GetFileName(file),
FilePath = file
};
await fileTransferUtility.UploadAsync(req);
// upload the tile files
TransferUtilityUploadDirectoryRequest request =
new TransferUtilityUploadDirectoryRequest()
{
KeyPrefix = filesKeyPrefix,
Directory = directory,
BucketName = asset.Project,
SearchOption = SearchOption.AllDirectories,
SearchPattern = "*.*"
};
await fileTransferUtility.UploadDirectoryAsync(request);
_logger.LogInformation($"finished upload for index {file} and directory {directory}");
return(true);
}
}
public async Task <ActionResult <OVEAssetModel> > Edit(string id, [Bind("Project,Name,Description,Id,StorageLocation,Processed,Service,AssetMeta,LastModified")]
OVEAssetModel oveAssetModel, [FromForm] IFormFile upload)
{
if (id != oveAssetModel.Id)
{
return(NotFound());
}
var oldAssetModel = await _context.AssetModels.FirstOrDefaultAsync(m => m.Id == id);
if (oldAssetModel == null)
{
return(NotFound());
}
// check someone is not tampering with the service after upload
if (upload == null && !_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, oveAssetModel.StorageLocation))
{
ModelState.AddModelError("Service", "Service does not support File Type");
}
// concurrent fields need to be updated by themselves and atomically.
bool need2UpdateProcessingState = oldAssetModel.ProcessingState != oveAssetModel.ProcessingState;
if (need2UpdateProcessingState)
{
oveAssetModel.ProcessingState = oldAssetModel.ProcessingState;
}
if (ModelState.IsValid)
{
try {
if (oldAssetModel.Project != oveAssetModel.Project)
{
await _fileOperations.Move(oldAssetModel, oveAssetModel); // todo not implemented
}
//stop EF from tracking the old version so that it will allow you to update the new version
_context.Entry(oldAssetModel).State = EntityState.Detached;
if (upload != null && upload.Length > 0)
{
if (!_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, upload))
{
ModelState.AddModelError("Service", "Service does not support File Type");
}
if (!await _fileOperations.Delete(oveAssetModel))
{
return(UnprocessableEntity("unable to delete old file"));
}
if (!await _fileOperations.Save(oveAssetModel, upload))
{
return(UnprocessableEntity("unable to save new file"));
}
need2UpdateProcessingState = true;
}
oveAssetModel.LastModified = DateTime.Now;
_context.Update(oveAssetModel);
await _context.SaveChangesAsync();
if (need2UpdateProcessingState)
{
oveAssetModel.ProcessingState = 0;
_context.Update(oveAssetModel);
await _context.SaveChangesAsync();
}
}
catch (DbUpdateConcurrencyException) {
if (!_context.AssetModels.Any(e => e.Id == oveAssetModel.Id))
{
return(NotFound());
}
_logger.LogError("Concurrency Error updating database ");
return(Conflict());
}
return(RedirectToAction(nameof(Index)));
}
return(this.FormatOrView(oveAssetModel));
}
public async Task Process(IAssetProcessingService <ArchiveProcessingStates> service, OVEAssetModel asset)
{
// 2) download it
string url = await service.GetAssetUri(asset);
string localUri = service.DownloadAsset(url, asset);
// 3) unzip and Upload
await service.UpdateStatus(asset, (int)ArchiveProcessingStates.Uploading);
List <string> files;
using (var s = File.OpenRead(localUri)) {
files = await UnZipAsset(asset, s);
}
// 4) set the meta data properly
asset.AssetMeta = JsonConvert.SerializeObject(files);
await UpdateAssetMeta(asset);
// 5) delete local files
_logger.LogInformation("about to delete files");
Directory.Delete(Path.GetDirectoryName(localUri), true);
// 6) Mark it as completed
await service.UpdateStatus(asset, (int)ArchiveProcessingStates.Processed);
}
