Esempio n. 1
0
        private async Task <List <string> > UnZipAsset(OVEAssetModel asset, Stream zipFile)
        {
            string prefixFolder = asset.StorageLocation.Split("/").FirstOrDefault() + "/";

            List <string> filesUploaded = new List <string>();

            var archive = new ZipArchive(zipFile);

            foreach (var entry in archive.Entries.Where(f => f.FullName.Contains(".")))
            {
                // only files
                var location = UnzipLocation(entry.FullName, prefixFolder);

                using (var file = entry.Open()) {
                    //raw unzipped streams do not have length so copy to memory stream
                    using (var ms = new MemoryStream()) {
                        await file.CopyToAsync(ms);

                        await _fileOps.Upload(asset.Project, location, ms);
                    }
                }

                filesUploaded.Add(location);
            }

            return(filesUploaded);
        }
        public string ResolveFileUrl(OVEAssetModel asset)
        {
            var url = _configuration.GetValue <string>(S3ClientServiceUrl).EnsureTrailingSlash()
                      + asset.Project + "/" + asset.StorageLocation;

            return(url);
        }
#pragma warning disable 1998
        public async Task <bool> Move(OVEAssetModel oldAsset, OVEAssetModel newAsset)
        {
#pragma warning restore 1998
            //todo this is hard because we might have to move between s3 buckets and that is complex
            // https://stackoverflow.com/questions/9664904/best-way-to-move-files-between-s3-buckets
            throw new NotImplementedException();
        }
Esempio n. 4
0
        /// <summary>
        /// Convert a numeric processing state into a user friendly message.
        /// </summary>
        /// <param name="asset">asset</param>
        /// <returns>user friendly message</returns>
        // ReSharper disable once UnusedMember.Global << used in view
        public string TranslateProcessingState(OVEAssetModel asset)
        {
            var service = GetService(asset.Service);

            if (service == null)
            {
                return("Unknown Service");
            }
            if (service.ProcessingStates.TryGetValue(asset.ProcessingState.ToString(), out string message))
            {
                return(message);
            }

            return(asset.ProcessingState == 0 ? "Unprocessed" : "unknown");
        }
Esempio n. 5
0
        // ReSharper disable once UnusedMember.Global << used in view
        public string GetViewUrl(OVEAssetModel model)
        {
            var service = GetService(model.Service);

            if (service == null)
            {
                return("");
            }

            var url = service.ViewIFrameUrl;

            return(string.IsNullOrWhiteSpace(url)
                ? ""
                : url.Replace("{id}", model.Id));
        }
        public async Task <bool> Save(OVEAssetModel asset, IFormFile upload)
        {
            _logger.LogInformation("about to upload " + asset);

            // set up the filename
            var ext = SanitizeExtension(Path.GetExtension(upload.FileName).ToLower());

            asset.StorageLocation = Guid.NewGuid() + "/" +
                                    SanitizeFilename(Path.GetFileNameWithoutExtension(upload.FileName), ext) + ext;

            try {
                using (var s3Client = GetS3Client(_configuration)) {
                    // find or create the bucket
                    var buckets = await s3Client.ListBucketsAsync();

                    if (buckets.Buckets.FirstOrDefault(b => b.BucketName == asset.Project) == null)
                    {
                        var res = await s3Client.PutBucketAsync(asset.Project);

                        if (res.HttpStatusCode != HttpStatusCode.OK)
                        {
                            throw new Exception("could not create bucket" + asset.Project);
                        }

                        var openBucketPolicy =
                            "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetBucketLocation\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::" +
                            asset.Project +
                            "\"]},{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetObject\"],\"Resource\":[\"arn:aws:s3:::" +
                            asset.Project + "/*\"]}]}";

                        await s3Client.PutBucketPolicyAsync(asset.Project, openBucketPolicy);

                        _logger.LogInformation("Created bucket " + asset.Project);
                    }

                    // upload the asset
                    await Upload(s3Client, asset.Project, asset.StorageLocation, upload);
                }

                _logger.LogInformation("uploaded " + asset);
                return(true);
            }
            catch (Exception e) {
                _logger.LogError(e, "Failed to upload file for " + asset);
                return(false);
            }
        }
        public string DownloadAsset(string url, OVEAssetModel asset)
        {
            // make temp directory
            // download url

            string localFile = Path.Combine(GetLocalAssetBasePath(), asset.StorageLocation);

            Directory.CreateDirectory(Path.GetDirectoryName(localFile));

            _logger.LogInformation("About to download to " + localFile);

            using (var client = new WebClient()) {
                client.DownloadFile(new Uri(url), localFile);
            }

            _logger.LogInformation("Finished downloading to " + localFile);

            return(localFile.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar));
        }
        public async Task <bool> Delete(OVEAssetModel asset)
        {
            _logger.LogInformation("about to delete file " + asset);
            try {
                using (var s3Client = GetS3Client(_configuration)) {
                    // delete folder containing the file and everything

                    ListObjectsResponse files = null;
                    while (files == null || files.S3Objects.Any())
                    {
                        if (files != null && files.S3Objects.Any())
                        {
                            foreach (var o in files.S3Objects)
                            {
                                await s3Client.DeleteObjectAsync(asset.Project, o.Key);
                            }
                        }

                        // find more files
                        files = await s3Client.ListObjectsAsync(new ListObjectsRequest {
                            BucketName = asset.Project,
                            Prefix     = asset.GetStorageGuid()
                        });
                    }

                    // if the bucket is empty then delete it
                    var res = await s3Client.ListObjectsAsync(asset.Project);

                    if (!res.S3Objects.Any())
                    {
                        await s3Client.DeleteBucketAsync(asset.Project);
                    }
                }

                _logger.LogInformation("deleted file on s3 correctly");
                return(true);
            }
            catch (Exception e) {
                _logger.LogError(e, "Failed to delete an s3 file for " + asset);
                return(false);
            }
        }
        /// <summary>
        /// This method calls the asset manger seek an asset to process
        /// </summary>
        /// <param name="state"></param>
        private async void ProcessAsset(object state)
        {
            if (!_processing.Wait(10))
            {
                _logger.LogInformation("Tried to fire an asset processor but too many threads already running");
                return;
            }
            _logger.LogInformation("About to seek an Asset to process");

            OVEAssetModel asset = null;

            try {
                // 1) get an Asset to process
                asset = await FindAssetToProcess();

                if (asset == null)
                {
                    _logger.LogInformation("no work for an asset Processor, running Processors = " +
                                           (_maxConcurrent - _processing.CurrentCount - 1));
                }
                else
                {
                    _logger.LogInformation("Found asset " + asset.Id);

                    await _processor.Process(this, asset);
                }
            } catch (Exception e) {
                _logger.LogError(e, "Exception in Asset Processing");

                try {
                    if (asset != null)
                    {
                        await UpdateStatus(asset, ProcessingErrorState, e.ToString());
                    }
                } catch (Exception ex) {
                    _logger.LogError(ex, "Exception in Asset Processing");
                }
            } finally {
                _processing.Release();
                _logger.LogInformation("released processing lock");
            }
        }
Esempio n. 10
0
        private async Task <string> GetAssetUri(OVEAssetModel asset)
        {
            string url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
                         _configuration.GetValue <string>("AssetUrlApi") +
                         asset.Id;

            using (var client = new HttpClient()) {
                var responseMessage = await client.GetAsync(url);

                if (responseMessage.StatusCode == HttpStatusCode.OK)
                {
                    var assetString = await responseMessage.Content.ReadAsStringAsync();

                    _logger.LogInformation("About to download asset from url " + assetString);
                    return(assetString);
                }
            }

            throw new Exception("Failed to get download URL for asset");
        }
Esempio n. 11
0
        private async Task <bool> UpdateAssetMeta(OVEAssetModel asset)
        {
            var url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
                      _configuration.GetValue <string>("UpdateMetaApi") +
                      asset.Id + ".json";

            _logger.LogInformation("Updating Asset Metadata");
            using (var client = new HttpClient()) {
                var body = new StringContent(asset.AssetMeta);
                body.Headers.ContentType = MediaTypeHeaderValue.Parse("text/plain");
                var responseMessage = await client.PostAsync(url, body);

                if (responseMessage.StatusCode != HttpStatusCode.OK)
                {
                    _logger.LogError("Failed to update the asset metadata " + responseMessage.StatusCode);
                    return(false);
                }
            }

            return(true);
        }
        public async Task <ActionResult <OVEAssetModel> > Create(
            [Bind("Project,Name,Description,Service,AssetMeta")] OVEAssetModel oveAssetModel,
            [FromForm] IFormFile upload)
        {
            // check if we have a file
            if (upload == null || upload.Length <= 0)
            {
                _logger.LogError("failed to upload a file");
                ModelState.AddModelError("Filename", "Failed to upload file");
            }
            else if (!_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, upload))
            {
                ModelState.AddModelError("Service", "Service does not support File Type");
            }

            if (ModelState.IsValid)
            {
                // then try and save it
                try {
                    await _fileOperations.Save(oveAssetModel, upload);

                    _logger.LogInformation("received and uploaded a file :) " + oveAssetModel.StorageLocation);
                }
                catch (Exception e) {
                    _logger.LogError(e, "failed to upload a file and write it to " + oveAssetModel.StorageLocation);
                    ModelState.AddModelError("StorageLocation", "Failed to upload file");
                }
            }

            if (ModelState.IsValid)
            {
                _context.Add(oveAssetModel);
                await _context.SaveChangesAsync();

                return(RedirectToAction(nameof(Index)));
            }

            return(this.FormatOrView(oveAssetModel));
        }
        private async Task <OVEAssetModel> FindAssetToProcess()
        {
            OVEAssetModel todo = null;
            string        url  = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
                                 _configuration.GetValue <string>("WorkItemApi") +
                                 _configuration.GetValue <string>("ServiceName") + ".json";

            _logger.LogInformation("about to get work item from url " + url);

            using (var client = new HttpClient()) {
                var responseMessage = await client.GetAsync(url);

                _logger.LogInformation("Response was " + responseMessage.StatusCode);
                if (responseMessage.StatusCode == HttpStatusCode.OK)
                {
                    var assetString = await responseMessage.Content.ReadAsStringAsync();

                    todo = JsonConvert.DeserializeObject <OVEAssetModel>(assetString);
                }
            }

            return(todo);
        }
        public async Task <bool> UpdateStatus(OVEAssetModel asset, int state, string errors = null) // todo int should be TV, see type constraint
        {
            var url = _configuration.GetValue <string>("AssetManagerHostUrl").RemoveTrailingSlash() +
                      _configuration.GetValue <string>("SetStateApi") +
                      asset.Id + "/" + state;

            if (errors != null)
            {
                url += "?message=" + Uri.EscapeDataString(errors);
            }

            _logger.LogInformation("Setting Asset State to " + state);
            using (var client = new HttpClient()) {
                var responseMessage = await client.PostAsync(url, new StringContent(""));

                if (responseMessage.StatusCode != HttpStatusCode.OK)
                {
                    _logger.LogError("Failed to set asset status " + responseMessage.StatusCode);
                    return(false);
                }
            }

            return(true);
        }
Esempio n. 15
0
        public async Task Process(IAssetProcessingService <ImageProcessingStates> service, OVEAssetModel asset)
        {
            // 2) download it
            string url = await service.GetAssetUri(asset);

            string localUri = service.DownloadAsset(url, asset);

            // 3) Create DZI file
            await service.UpdateStatus(asset, (int)ImageProcessingStates.CreatingDZI);

            var res = ProcessFile(localUri);

            _logger.LogInformation("Processed file " + res);

            // 4) Upload it
            await service.UpdateStatus(asset, (int)ImageProcessingStates.Uploading);

            await _fileOps.UploadIndexFileAndDirectory(Path.ChangeExtension(localUri, ".dzi"),
                                                       Path.ChangeExtension(localUri, ".dzi").Replace(".dzi", "_files/"),
                                                       asset);

            // 5) delete local files
            _logger.LogInformation("about to delete files");
            Directory.Delete(Path.GetDirectoryName(localUri), true);

            // 6) Mark it as completed
            await service.UpdateStatus(asset, (int)ImageProcessingStates.Processed);
        }
        public async Task <bool> UploadIndexFileAndDirectory(string file, string directory, OVEAssetModel asset)
        {
            _logger.LogInformation($"about to upload index {file} and directory {directory}");

            using (var fileTransferUtility = new TransferUtility(GetS3Client(_configuration))) {
                // upload the index file
                var assetRootFolder = Path.GetDirectoryName(asset.StorageLocation);

                var filesKeyPrefix = assetRootFolder + "/" + new DirectoryInfo(directory).Name + "/"; // upload to the right folder

                TransferUtilityUploadRequest req = new TransferUtilityUploadRequest {
                    BucketName = asset.Project,
                    Key        = assetRootFolder + "/" + Path.GetFileName(file),
                    FilePath   = file
                };
                await fileTransferUtility.UploadAsync(req);

                // upload the tile files

                TransferUtilityUploadDirectoryRequest request =
                    new TransferUtilityUploadDirectoryRequest()
                {
                    KeyPrefix     = filesKeyPrefix,
                    Directory     = directory,
                    BucketName    = asset.Project,
                    SearchOption  = SearchOption.AllDirectories,
                    SearchPattern = "*.*"
                };

                await fileTransferUtility.UploadDirectoryAsync(request);

                _logger.LogInformation($"finished upload for index {file} and directory {directory}");

                return(true);
            }
        }
        public async Task <ActionResult <OVEAssetModel> > Edit(string id, [Bind("Project,Name,Description,Id,StorageLocation,Processed,Service,AssetMeta,LastModified")]
                                                               OVEAssetModel oveAssetModel, [FromForm] IFormFile upload)
        {
            if (id != oveAssetModel.Id)
            {
                return(NotFound());
            }

            var oldAssetModel = await _context.AssetModels.FirstOrDefaultAsync(m => m.Id == id);

            if (oldAssetModel == null)
            {
                return(NotFound());
            }

            // check someone is not tampering with the service after upload
            if (upload == null && !_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, oveAssetModel.StorageLocation))
            {
                ModelState.AddModelError("Service", "Service does not support File Type");
            }

            // concurrent fields need to be updated by themselves and atomically.
            bool need2UpdateProcessingState = oldAssetModel.ProcessingState != oveAssetModel.ProcessingState;

            if (need2UpdateProcessingState)
            {
                oveAssetModel.ProcessingState = oldAssetModel.ProcessingState;
            }

            if (ModelState.IsValid)
            {
                try {
                    if (oldAssetModel.Project != oveAssetModel.Project)
                    {
                        await _fileOperations.Move(oldAssetModel, oveAssetModel); // todo not implemented
                    }
                    //stop EF from tracking the old version so that it will allow you to update the new version
                    _context.Entry(oldAssetModel).State = EntityState.Detached;

                    if (upload != null && upload.Length > 0)
                    {
                        if (!_serviceRepository.ValidateServiceChoice(oveAssetModel.Service, upload))
                        {
                            ModelState.AddModelError("Service", "Service does not support File Type");
                        }

                        if (!await _fileOperations.Delete(oveAssetModel))
                        {
                            return(UnprocessableEntity("unable to delete old file"));
                        }

                        if (!await _fileOperations.Save(oveAssetModel, upload))
                        {
                            return(UnprocessableEntity("unable to save new file"));
                        }
                        need2UpdateProcessingState = true;
                    }
                    oveAssetModel.LastModified = DateTime.Now;
                    _context.Update(oveAssetModel);

                    await _context.SaveChangesAsync();

                    if (need2UpdateProcessingState)
                    {
                        oveAssetModel.ProcessingState = 0;
                        _context.Update(oveAssetModel);
                        await _context.SaveChangesAsync();
                    }
                }
                catch (DbUpdateConcurrencyException) {
                    if (!_context.AssetModels.Any(e => e.Id == oveAssetModel.Id))
                    {
                        return(NotFound());
                    }
                    _logger.LogError("Concurrency Error updating database ");
                    return(Conflict());
                }

                return(RedirectToAction(nameof(Index)));
            }

            return(this.FormatOrView(oveAssetModel));
        }
Esempio n. 18
0
        public async Task Process(IAssetProcessingService <ArchiveProcessingStates> service, OVEAssetModel asset)
        {
            // 2) download it
            string url = await service.GetAssetUri(asset);

            string localUri = service.DownloadAsset(url, asset);

            // 3) unzip and Upload
            await service.UpdateStatus(asset, (int)ArchiveProcessingStates.Uploading);

            List <string> files;

            using (var s = File.OpenRead(localUri)) {
                files = await UnZipAsset(asset, s);
            }

            // 4) set the meta data properly

            asset.AssetMeta = JsonConvert.SerializeObject(files);
            await UpdateAssetMeta(asset);

            // 5) delete local files
            _logger.LogInformation("about to delete files");
            Directory.Delete(Path.GetDirectoryName(localUri), true);

            // 6) Mark it as completed
            await service.UpdateStatus(asset, (int)ArchiveProcessingStates.Processed);
        }