Exemplo n.º 1
0
        /// <summary>
        /// ADMIN ONLY - Removes images from the debug bucket
        /// </summary>
        public static void ADMINCleanUpDebug()
        {
            using (IAmazonS3 s3 = AWSConfiguration.S3Client())
            {
                ListObjectsRequest request = new ListObjectsRequest()
                {
                    BucketName = AWSConfiguration.S3BucketNameDebug
                };

                do
                {
                    ListObjectsResponse response = s3.ListObjects(request);

                    foreach (S3Object o in response.S3Objects)
                    {
                        DeleteObjectRequest dor = new DeleteObjectRequest()
                        {
                            BucketName = AWSConfiguration.S3BucketNameDebug, Key = o.Key
                        };
                        s3.DeleteObject(dor);
                    }

                    // If response is truncated, set the marker to get the next
                    // set of keys.
                    if (response.IsTruncated)
                    {
                        request.Marker = response.NextMarker;
                    }
                    else
                    {
                        request = null;
                    }
                } while (request != null);
            }
        }
Exemplo n.º 2
0
        /// <summary>
        /// Moves an image (well, any object, really) from one key to another
        /// </summary>
        /// <param name="szSrc">Source path (key)</param>
        /// <param name="szDst">Destination path (key)</param>
        public static void MoveImageOnS3(string szSrc, string szDst)
        {
            try
            {
                using (IAmazonS3 s3 = AWSConfiguration.S3Client())
                {
                    CopyObjectRequest   cor = new CopyObjectRequest();
                    DeleteObjectRequest dor = new DeleteObjectRequest();
                    cor.SourceBucket   = cor.DestinationBucket = dor.BucketName = AWSConfiguration.CurrentS3Bucket;
                    cor.DestinationKey = szDst;
                    cor.SourceKey      = dor.Key = szSrc;
                    cor.CannedACL      = S3CannedACL.PublicRead;
                    cor.StorageClass   = S3StorageClass.Standard; // vs. reduced

                    s3.CopyObject(cor);
                    s3.DeleteObject(dor);
                }
            }
            catch (AmazonS3Exception ex)
            {
                util.NotifyAdminEvent("Error moving image on S3", String.Format(CultureInfo.InvariantCulture, "Error moving from key\r\n{0}to\r\n{1}\r\n\r\n{2}", szSrc, szDst, WrapAmazonS3Exception(ex)), ProfileRoles.maskSiteAdminOnly);
                throw new MyFlightbookException(String.Format(CultureInfo.InvariantCulture, "Error moving file on S3: Request address:{0}, Message:{1}", WrapAmazonS3Exception(ex), ex.Message));
            }
            catch (Exception ex)
            {
                throw new MyFlightbookException("Unknown error moving image on S3: " + ex.Message);
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Deletes the image from S3.  The actual operation happens asynchronously; the result is not captured.
        /// </summary>
        /// <param name="mfbii">The image to delete</param>
        public static void DeleteImageOnS3(MFBImageInfo mfbii)
        {
            if (mfbii == null)
            {
                return;
            }

            try
            {
                DeleteObjectRequest dor = new DeleteObjectRequest()
                {
                    BucketName = AWSConfiguration.CurrentS3Bucket,
                    Key        = mfbii.S3Key
                };

                new Thread(new ThreadStart(() =>
                {
                    try
                    {
                        using (IAmazonS3 s3 = AWSConfiguration.S3Client())
                        {
                            s3.DeleteObject(dor);
                            if (mfbii.ImageType == MFBImageInfo.ImageFileType.S3VideoMP4)
                            {
                                // Delete the thumbnail too.
                                string szs3Key = mfbii.S3Key;
                                dor.Key        = szs3Key.Replace(Path.GetFileName(szs3Key), MFBImageInfo.ThumbnailPrefixVideo + Path.GetFileNameWithoutExtension(szs3Key) + "00001" + FileExtensions.JPG);
                                s3.DeleteObject(dor);
                            }
                        }
                    }
                    catch (Exception ex) when(ex is AmazonS3Exception)
                    {
                    }
                }
                                           )).Start();
            }
            catch (AmazonS3Exception ex)
            {
                throw new MyFlightbookException(String.Format(CultureInfo.InvariantCulture, "Error deleting file on S3: {0}", WrapAmazonS3Exception(ex)), ex.InnerException);
            }
            catch (Exception ex)
            {
                throw new MyFlightbookException("Unknown error deleting image on S3: " + ex.Message);
            }
        }
Exemplo n.º 4
0
        /// <summary>
        /// ADMIN ONLY - Remove images from S3 LIVE BUCKET that are orphaned (no reference from live site)
        /// </summary>
        /// <param name="ic"></param>
        /// <param name="handleS3Object"></param>
        public static void ADMINDeleteS3Orphans(MFBImageInfo.ImageClass ic, Action <long, long, long, long> onSummary, Action onS3EnumDone, Func <string, int, bool> onDelete)
        {
            string szBasePath = MFBImageInfo.BasePathFromClass(ic);

            if (szBasePath.StartsWith("/", StringComparison.InvariantCultureIgnoreCase))
            {
                szBasePath = szBasePath.Substring(1);
            }

            Regex r = new Regex(String.Format(CultureInfo.InvariantCulture, "{0}(.*)/(.*)(\\.jpg|\\.jpeg|\\.pdf|\\.s3pdf|\\.mp4)", szBasePath), RegexOptions.IgnoreCase);

            long cBytesToFree  = 0;
            long cBytesOnS3    = 0;
            long cFilesOnS3    = 0;
            long cOrphansFound = 0;

            using (IAmazonS3 s3 = AWSConfiguration.S3Client())
            {
                ListObjectsRequest request = new ListObjectsRequest()
                {
                    BucketName = AWSConfiguration.S3BucketName, Prefix = szBasePath
                };

                /*
                 * Need to get the files from S3 FIRST, otherwise there is a race condition
                 * I.e., if we get files from the DB, then get files from S3, a file could be added to the db AFTER our query
                 * but BEFORE we retrieve it from the S3 listing, and we will thus treat it as an orphan and delete it.
                 * But since the file is put into the DB before being moved to S3, if we get all the S3 files and THEN
                 * get the DB references, we will always have a subset of the valid S3 files, which prevents a false positive
                 * orphan identification.
                 */

                List <S3Object> lstS3Objects = new List <S3Object>();
                // Get the list of S3 objects
                do
                {
                    ListObjectsResponse response = s3.ListObjects(request);

                    cFilesOnS3 += response.S3Objects.Count;
                    foreach (S3Object o in response.S3Objects)
                    {
                        cBytesOnS3 += o.Size;
                        lstS3Objects.Add(o);
                    }

                    // If response is truncated, set the marker to get the next
                    // set of keys.
                    if (response.IsTruncated)
                    {
                        request.Marker = response.NextMarker;
                    }
                    else
                    {
                        request = null;
                    }
                } while (request != null);

                onS3EnumDone?.Invoke();

                // Now get all of the images in the class and do orphan detection
                Dictionary <string, MFBImageInfo> dictDBResults = MFBImageInfo.AllImagesForClass(ic);

                long  cOrphansLikely = Math.Max(lstS3Objects.Count - dictDBResults.Keys.Count, 1);
                Regex rGuid          = new Regex(String.Format(CultureInfo.InvariantCulture, "^({0})?([^_]*).*", MFBImageInfo.ThumbnailPrefixVideo), RegexOptions.Compiled); // for use below in extracting GUIDs from video thumbnail and video file names.
                lstS3Objects.ForEach((o) =>
                {
                    Match m = r.Match(o.Key);
                    if (m.Groups.Count < 3)
                    {
                        return;
                    }

                    string szKey  = m.Groups[1].Value;
                    string szName = m.Groups[2].Value;
                    string szExt  = m.Groups[3].Value;

                    bool fPDF      = (String.Compare(szExt, FileExtensions.PDF, StringComparison.InvariantCultureIgnoreCase) == 0);
                    bool fVid      = (String.Compare(szExt, FileExtensions.MP4, StringComparison.InvariantCultureIgnoreCase) == 0);
                    bool fVidThumb = Regex.IsMatch(szExt, FileExtensions.RegExpImageFileExtensions) && szName.StartsWith(MFBImageInfo.ThumbnailPrefixVideo, StringComparison.InvariantCultureIgnoreCase);
                    bool fJpg      = (String.Compare(szExt, FileExtensions.JPG, StringComparison.InvariantCultureIgnoreCase) == 0 || String.Compare(szExt, FileExtensions.JPEG, StringComparison.InvariantCultureIgnoreCase) == 0);

                    string szThumb = string.Empty;
                    if (fPDF)
                    {
                        szThumb = szName + FileExtensions.S3PDF;
                    }
                    else if (fVid || fVidThumb)
                    {
                        // This is a bit of a hack, but we have two files on the server for a video, neither of which precisely matches what's in the database.
                        // The video file is {guid}.mp4 or {guid}_.mp4, the thumbnail on S3 is v_{guid}_0001.jpg.
                        // So we grab the GUID and see if we have a database entry matching that guid.
                        Match mGuid       = rGuid.Match(szName);
                        szThumb           = (mGuid.Groups.Count >= 3) ? mGuid.Groups[2].Value : szName + szExt;
                        string szMatchKey = dictDBResults.Keys.FirstOrDefault(skey => skey.Contains(szThumb));
                        if (!String.IsNullOrEmpty(szMatchKey))  // leave it in the dictionary - don't remove it - because we may yet hit the Mp4 or the JPG.
                        {
                            return;
                        }
                    }
                    else if (fJpg)
                    {
                        szThumb = MFBImageInfo.ThumbnailPrefix + szName + szExt;
                    }

                    string szPrimary = MFBImageInfo.PrimaryKeyForValues(ic, szKey, szThumb);

                    // if it is found, super - remove it from the dictionary (for performance) and return
                    if (dictDBResults.ContainsKey(szPrimary))
                    {
                        dictDBResults.Remove(szPrimary);
                    }
                    else
                    {
                        cOrphansFound++;
                        cBytesToFree += o.Size;
                        if (onDelete(o.Key, (int)((100 * cOrphansFound) / cOrphansLikely)))
                        {
                            // Make sure that the item
                            DeleteObjectRequest dor = new DeleteObjectRequest()
                            {
                                BucketName = AWSConfiguration.S3BucketName, Key = o.Key
                            };
                            s3.DeleteObject(dor);
                        }
                    }
                });

                onSummary?.Invoke(cFilesOnS3, cBytesOnS3, cOrphansFound, cBytesToFree);
            }
        }
Exemplo n.º 5
0
        /// <summary>
        /// Uploads a video for transcoding.  Deletes the source file named in szFileName
        /// </summary>
        /// <param name="szFileName">Filename of the input stream</param>
        /// <param name="szContentType">Content type of the video file</param>
        /// <param name="szBucket">Bucket to use.  Specified as a parameter because CurrentBucket might return the wrong value when called on a background thread</param>
        /// <param name="szPipelineID">PipelineID from amazon</param>
        /// <param name="mfbii">The MFBImageInfo that encapsulates the video</param>
        public void UploadVideo(string szFileName, string szContentType, string szBucket, string szPipelineID, MFBImageInfo mfbii)
        {
            if (mfbii == null)
            {
                throw new ArgumentNullException(nameof(mfbii));
            }

            using (var etsClient = AWSConfiguration.ElasticTranscoderClient())
            {
                string szGuid     = Guid.NewGuid() + MFBImageInfo.szNewS3KeySuffix;
                string szBasePath = mfbii.VirtualPath.StartsWith("/", StringComparison.Ordinal) ? mfbii.VirtualPath.Substring(1) : mfbii.VirtualPath;
                string szBaseFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}", szBasePath, szGuid);

                PutObjectRequest por = new PutObjectRequest()
                {
                    BucketName      = szBucket,
                    ContentType     = szContentType,
                    AutoCloseStream = true,
                    InputStream     = new FileStream(szFileName, FileMode.Open, FileAccess.Read),
                    Key             = szBaseFile + FileExtensions.VidInProgress,
                    CannedACL       = S3CannedACL.PublicRead,
                    StorageClass    = S3StorageClass.Standard // vs. reduced
                };

                lock (lockObject)
                {
                    try
                    {
                        using (por.InputStream)
                        {
                            using (IAmazonS3 s3 = AWSConfiguration.S3Client())
                                s3.PutObject(por);
                            File.Delete(szFileName);
                        }
                    }
                    catch (Exception ex)
                    {
                        util.NotifyAdminEvent("Error putting video file", ex.Message, ProfileRoles.maskSiteAdminOnly);
                        throw;
                    }
                }

                var job = etsClient.CreateJob(new CreateJobRequest()
                {
                    PipelineId = szPipelineID,
                    Input      = new JobInput()
                    {
                        AspectRatio = "auto",
                        Container   = "auto",
                        FrameRate   = "auto",
                        Interlaced  = "auto",
                        Resolution  = "auto",
                        Key         = por.Key,
                    },
                    Output = new CreateJobOutput()
                    {
                        ThumbnailPattern = String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}{3}", szBasePath, MFBImageInfo.ThumbnailPrefixVideo, szGuid, "{count}"),
                        Rotate           = "auto",
                        // Generic 720p: Go to http://docs.aws.amazon.com/elastictranscoder/latest/developerguide/create-job.html#PresetId to see a list of some
                        // of the support presets or call the ListPresets operation to get the full list of available presets
                        // PresetId = "1351620000000-000010",
                        PresetId = "1423799228749-hsj7ba",
                        Key      = szBaseFile + FileExtensions.MP4
                    }
                });
                if (job != null)
                {
                    new PendingVideo(szGuid, job.Job.Id, mfbii.Comment, mfbii.Class, mfbii.Key, szBucket).Commit();
                }
            }
        }
Exemplo n.º 6
0
        /// <summary>
        /// Moves the image to Amazon
        /// </summary>
        /// <param name="por"></param>
        /// <param name="mfbii">The object to move.</param>
        public void MoveByRequest(PutObjectRequest por, MFBImageInfo mfbii)
        {
            if (mfbii == null)
            {
                throw new ArgumentNullException(nameof(mfbii));
            }
            if (por == null)
            {
                throw new ArgumentNullException(nameof(por));
            }
            lock (lockObject)
            {
                try
                {
                    using (IAmazonS3 s3 = AWSConfiguration.S3Client())
                    {
                        PutObjectResponse s3r = null;
                        using (por.InputStream)
                        {
                            s3r = s3.PutObject(por);
                        }
                        if (s3r != null)
                        {
                            switch (mfbii.ImageType)
                            {
                            case MFBImageInfo.ImageFileType.JPEG:
                                File.Delete(mfbii.PhysicalPathFull);
                                break;

                            case MFBImageInfo.ImageFileType.PDF:
                            {
                                try
                                {
                                    if (String.IsNullOrEmpty(mfbii.Comment))
                                    {
                                        mfbii.Comment = mfbii.ThumbnailFile;
                                    }
                                    mfbii.ImageType = MFBImageInfo.ImageFileType.S3PDF;
                                    mfbii.RenameLocalFile(mfbii.ThumbnailFile.Replace(FileExtensions.PDF, FileExtensions.S3PDF));

                                    // Write the comment to the resulting file.
                                    using (FileStream fs = File.OpenWrite(mfbii.PhysicalPathThumbnail))
                                    {
                                        fs.SetLength(0);
                                        byte[] rgBytes = Encoding.UTF8.GetBytes(mfbii.Comment.ToCharArray());
                                        fs.Write(rgBytes, 0, rgBytes.Length);
                                    }
                                }
                                catch (Exception ex) when(ex is UnauthorizedAccessException || ex is FileNotFoundException || ex is IOException)
                                {
                                    mfbii.ImageType = MFBImageInfo.ImageFileType.PDF;
                                }
                            }
                            break;

                            default:
                                break;
                            }
                            // ALWAYS update the db
                            mfbii.UpdateDBLocation(false);
                        }
                    }
                }
                catch (AmazonS3Exception ex)
                {
                    throw new MyFlightbookException(WrapAmazonS3Exception(ex), ex);
                }
            }
        }
Exemplo n.º 7
0
        /// <summary>
        /// Creates the thumbnail from a completed video process.  Sets the width/height in the process.
        /// </summary>
        /// <param name="szBasePath">The base path for the video object</param>
        /// <param name="szPhysicalPath">The filename to use for the resulting thumbnail image</param>
        public void InitThumbnail(string szBasePath, string szPhysicalPath)
        {
            if (szBasePath == null)
            {
                throw new ArgumentNullException(nameof(szBasePath));
            }

            string szThumbFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}00001{2}", MFBImageInfo.ThumbnailPrefixVideo, GUID, FileExtensions.JPG);

            if (szBasePath.StartsWith("/", StringComparison.Ordinal))
            {
                szBasePath = szBasePath.Substring(1);
            }
            string srcFile = szBasePath + szThumbFile;

            // Copy the thumbnail over
            using (IAmazonS3 s3 = AWSConfiguration.S3Client())
            {
                try
                {
                    using (GetObjectResponse gor = s3.GetObject(new GetObjectRequest()
                    {
                        BucketName = Bucket, Key = srcFile
                    }))
                    {
                        if (gor != null && gor.ResponseStream != null)
                        {
#pragma warning disable IDISP007 // Don't dispose injected
                            // Amazon documents EXPLICITLY say we should wrap in a using block.  See https://docs.aws.amazon.com/sdkfornet1/latest/apidocs/html/P_Amazon_S3_Model_S3Response_ResponseStream.htm
                            using (gor.ResponseStream)
#pragma warning restore IDISP007 // Don't dispose injected.
                            {
                                using (System.Drawing.Image image = MFBImageInfo.DrawingCompatibleImageFromStream(gor.ResponseStream))
                                {
                                    Info inf = MFBImageInfo.InfoFromImage(image);
                                    // save the thumbnail locally.
                                    inf.ImageDescription = Comment;

                                    Bitmap bmp = MFBImageInfo.BitmapFromImage(inf.Image, MFBImageInfo.ThumbnailHeight, MFBImageInfo.ThumbnailWidth);
                                    ThumbWidth  = bmp.Width;
                                    ThumbHeight = bmp.Height;

                                    using (bmp)
                                    {
                                        // get all properties of the original image and copy them to the new image.  This should include the annotation (above)
                                        foreach (PropertyItem pi in inf.Image.PropertyItems)
                                        {
                                            bmp.SetPropertyItem(pi);
                                        }

                                        bmp.Save(szPhysicalPath, ImageFormat.Jpeg);
                                    }
                                }
                            }
                        }
                    }
                }
                catch (AmazonS3Exception)
                {
                    // Thumbnail was not found - audio file perhaps?  Use the generic audio file.
                    System.IO.File.Copy(HostingEnvironment.MapPath("~/images/audio.png"), szPhysicalPath);
                }


                // clean up the folder on S3 - anything that has the GUID but not .mp4 in it or the thumbnail in it.  (Save space!)  i.e., delete excess thumbnails and the source video file.
                List <S3Object>    lstS3Objects = new List <S3Object>();
                ListObjectsRequest loRequest    = new ListObjectsRequest()
                {
                    BucketName = Bucket, Prefix = szBasePath
                };
                // Get the list of S3 objects
                do
                {
                    ListObjectsResponse response = s3.ListObjects(loRequest);
                    foreach (S3Object o in response.S3Objects)
                    {
                        if (o.Key.Contains(GUID) && !o.Key.Contains(FileExtensions.MP4) && !o.Key.Contains(szThumbFile))
                        {
                            lstS3Objects.Add(o);
                        }
                    }

                    // If response is truncated, set the marker to get the next
                    // set of keys.
                    if (response.IsTruncated)
                    {
                        loRequest.Marker = response.NextMarker;
                    }
                    else
                    {
                        loRequest = null;
                    }
                } while (loRequest != null);

                lstS3Objects.ForEach((o) =>
                {
                    s3.DeleteObject(new DeleteObjectRequest()
                    {
                        BucketName = Bucket, Key = o.Key
                    });
                });
            }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Creates the thumbnail from a completed video process.  Sets the width/height in the process.
        /// </summary>
        /// <param name="szBasePath">The base path for the video object</param>
        /// <param name="szPhysicalPath">The filename to use for the resulting thumbnail image</param>
        public void InitThumbnail(string szBasePath, string szPhysicalPath)
        {
            if (szBasePath == null)
            {
                throw new ArgumentNullException(nameof(szBasePath));
            }

            string szThumbFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}00001{2}", MFBImageInfo.ThumbnailPrefixVideo, GUID, FileExtensions.JPG);

            if (szBasePath.StartsWith("/", StringComparison.Ordinal))
            {
                szBasePath = szBasePath.Substring(1);
            }
            string srcFile = szBasePath + szThumbFile;

            // Copy the thumbnail over
            using (IAmazonS3 s3 = AWSConfiguration.S3Client())
            {
                GetObjectResponse gor = s3.GetObject(new GetObjectRequest()
                {
                    BucketName = Bucket, Key = srcFile
                });
                if (gor != null && gor.ResponseStream != null)
                {
                    using (gor.ResponseStream)
                    {
                        using (System.Drawing.Image image = MFBImageInfo.DrawingCompatibleImageFromStream(gor.ResponseStream))
                        {
                            Info inf = MFBImageInfo.InfoFromImage(image);

                            // save the thumbnail locally.
                            using (inf.Image)
                            {
                                inf.ImageDescription = Comment;

                                Bitmap bmp = MFBImageInfo.BitmapFromImage(inf.Image, MFBImageInfo.ThumbnailHeight, MFBImageInfo.ThumbnailWidth);
                                ThumbWidth  = bmp.Width;
                                ThumbHeight = bmp.Height;

                                using (bmp)
                                {
                                    // get all properties of the original image and copy them to the new image.  This should include the annotation (above)
                                    foreach (PropertyItem pi in inf.Image.PropertyItems)
                                    {
                                        bmp.SetPropertyItem(pi);
                                    }

                                    bmp.Save(szPhysicalPath, ImageFormat.Jpeg);
                                }
                            }
                        }
                    }
                }

                // clean up the folder on S3 - anything that has the GUID but not .mp4 in it or the thumbnail in it.  (Save space!)  i.e., delete excess thumbnails and the source video file.
                List <S3Object>    lstS3Objects = new List <S3Object>();
                ListObjectsRequest loRequest    = new ListObjectsRequest()
                {
                    BucketName = Bucket, Prefix = szBasePath
                };
                // Get the list of S3 objects
                do
                {
                    ListObjectsResponse response = s3.ListObjects(loRequest);
                    foreach (S3Object o in response.S3Objects)
                    {
                        if (o.Key.Contains(GUID) && !o.Key.Contains(FileExtensions.MP4) && !o.Key.Contains(szThumbFile))
                        {
                            lstS3Objects.Add(o);
                        }
                    }

                    // If response is truncated, set the marker to get the next
                    // set of keys.
                    if (response.IsTruncated)
                    {
                        loRequest.Marker = response.NextMarker;
                    }
                    else
                    {
                        loRequest = null;
                    }
                } while (loRequest != null);

                lstS3Objects.ForEach((o) =>
                {
                    s3.DeleteObject(new DeleteObjectRequest()
                    {
                        BucketName = Bucket, Key = o.Key
                    });
                });
            }
        }