/// <summary>
/// ADMIN ONLY - Removes images from the debug bucket
/// </summary>
public static void ADMINCleanUpDebug()
{
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = AWSConfiguration.S3BucketNameDebug
};
do
{
ListObjectsResponse response = s3.ListObjects(request);
foreach (S3Object o in response.S3Objects)
{
DeleteObjectRequest dor = new DeleteObjectRequest()
{
BucketName = AWSConfiguration.S3BucketNameDebug, Key = o.Key
};
s3.DeleteObject(dor);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
}
}
/// <summary>
/// Moves an image (well, any object, really) from one key to another
/// </summary>
/// <param name="szSrc">Source path (key)</param>
/// <param name="szDst">Destination path (key)</param>
public static void MoveImageOnS3(string szSrc, string szDst)
{
try
{
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
CopyObjectRequest cor = new CopyObjectRequest();
DeleteObjectRequest dor = new DeleteObjectRequest();
cor.SourceBucket = cor.DestinationBucket = dor.BucketName = AWSConfiguration.CurrentS3Bucket;
cor.DestinationKey = szDst;
cor.SourceKey = dor.Key = szSrc;
cor.CannedACL = S3CannedACL.PublicRead;
cor.StorageClass = S3StorageClass.Standard; // vs. reduced
s3.CopyObject(cor);
s3.DeleteObject(dor);
}
}
catch (AmazonS3Exception ex)
{
util.NotifyAdminEvent("Error moving image on S3", String.Format(CultureInfo.InvariantCulture, "Error moving from key\r\n{0}to\r\n{1}\r\n\r\n{2}", szSrc, szDst, WrapAmazonS3Exception(ex)), ProfileRoles.maskSiteAdminOnly);
throw new MyFlightbookException(String.Format(CultureInfo.InvariantCulture, "Error moving file on S3: Request address:{0}, Message:{1}", WrapAmazonS3Exception(ex), ex.Message));
}
catch (Exception ex)
{
throw new MyFlightbookException("Unknown error moving image on S3: " + ex.Message);
}
}
/// <summary>
/// Deletes the image from S3. The actual operation happens asynchronously; the result is not captured.
/// </summary>
/// <param name="mfbii">The image to delete</param>
public static void DeleteImageOnS3(MFBImageInfo mfbii)
{
if (mfbii == null)
{
return;
}
try
{
DeleteObjectRequest dor = new DeleteObjectRequest()
{
BucketName = AWSConfiguration.CurrentS3Bucket,
Key = mfbii.S3Key
};
new Thread(new ThreadStart(() =>
{
try
{
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
s3.DeleteObject(dor);
if (mfbii.ImageType == MFBImageInfo.ImageFileType.S3VideoMP4)
{
// Delete the thumbnail too.
string szs3Key = mfbii.S3Key;
dor.Key = szs3Key.Replace(Path.GetFileName(szs3Key), MFBImageInfo.ThumbnailPrefixVideo + Path.GetFileNameWithoutExtension(szs3Key) + "00001" + FileExtensions.JPG);
s3.DeleteObject(dor);
}
}
}
catch (Exception ex) when(ex is AmazonS3Exception)
{
}
}
)).Start();
}
catch (AmazonS3Exception ex)
{
throw new MyFlightbookException(String.Format(CultureInfo.InvariantCulture, "Error deleting file on S3: {0}", WrapAmazonS3Exception(ex)), ex.InnerException);
}
catch (Exception ex)
{
throw new MyFlightbookException("Unknown error deleting image on S3: " + ex.Message);
}
}
/// <summary>
/// ADMIN ONLY - Remove images from S3 LIVE BUCKET that are orphaned (no reference from live site)
/// </summary>
/// <param name="ic"></param>
/// <param name="handleS3Object"></param>
public static void ADMINDeleteS3Orphans(MFBImageInfo.ImageClass ic, Action <long, long, long, long> onSummary, Action onS3EnumDone, Func <string, int, bool> onDelete)
{
string szBasePath = MFBImageInfo.BasePathFromClass(ic);
if (szBasePath.StartsWith("/", StringComparison.InvariantCultureIgnoreCase))
{
szBasePath = szBasePath.Substring(1);
}
Regex r = new Regex(String.Format(CultureInfo.InvariantCulture, "{0}(.*)/(.*)(\\.jpg|\\.jpeg|\\.pdf|\\.s3pdf|\\.mp4)", szBasePath), RegexOptions.IgnoreCase);
long cBytesToFree = 0;
long cBytesOnS3 = 0;
long cFilesOnS3 = 0;
long cOrphansFound = 0;
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = AWSConfiguration.S3BucketName, Prefix = szBasePath
};
/*
* Need to get the files from S3 FIRST, otherwise there is a race condition
* I.e., if we get files from the DB, then get files from S3, a file could be added to the db AFTER our query
* but BEFORE we retrieve it from the S3 listing, and we will thus treat it as an orphan and delete it.
* But since the file is put into the DB before being moved to S3, if we get all the S3 files and THEN
* get the DB references, we will always have a subset of the valid S3 files, which prevents a false positive
* orphan identification.
*/
List <S3Object> lstS3Objects = new List <S3Object>();
// Get the list of S3 objects
do
{
ListObjectsResponse response = s3.ListObjects(request);
cFilesOnS3 += response.S3Objects.Count;
foreach (S3Object o in response.S3Objects)
{
cBytesOnS3 += o.Size;
lstS3Objects.Add(o);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
onS3EnumDone?.Invoke();
// Now get all of the images in the class and do orphan detection
Dictionary <string, MFBImageInfo> dictDBResults = MFBImageInfo.AllImagesForClass(ic);
long cOrphansLikely = Math.Max(lstS3Objects.Count - dictDBResults.Keys.Count, 1);
Regex rGuid = new Regex(String.Format(CultureInfo.InvariantCulture, "^({0})?([^_]*).*", MFBImageInfo.ThumbnailPrefixVideo), RegexOptions.Compiled); // for use below in extracting GUIDs from video thumbnail and video file names.
lstS3Objects.ForEach((o) =>
{
Match m = r.Match(o.Key);
if (m.Groups.Count < 3)
{
return;
}
string szKey = m.Groups[1].Value;
string szName = m.Groups[2].Value;
string szExt = m.Groups[3].Value;
bool fPDF = (String.Compare(szExt, FileExtensions.PDF, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVid = (String.Compare(szExt, FileExtensions.MP4, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVidThumb = Regex.IsMatch(szExt, FileExtensions.RegExpImageFileExtensions) && szName.StartsWith(MFBImageInfo.ThumbnailPrefixVideo, StringComparison.InvariantCultureIgnoreCase);
bool fJpg = (String.Compare(szExt, FileExtensions.JPG, StringComparison.InvariantCultureIgnoreCase) == 0 || String.Compare(szExt, FileExtensions.JPEG, StringComparison.InvariantCultureIgnoreCase) == 0);
string szThumb = string.Empty;
if (fPDF)
{
szThumb = szName + FileExtensions.S3PDF;
}
else if (fVid || fVidThumb)
{
// This is a bit of a hack, but we have two files on the server for a video, neither of which precisely matches what's in the database.
// The video file is {guid}.mp4 or {guid}_.mp4, the thumbnail on S3 is v_{guid}_0001.jpg.
// So we grab the GUID and see if we have a database entry matching that guid.
Match mGuid = rGuid.Match(szName);
szThumb = (mGuid.Groups.Count >= 3) ? mGuid.Groups[2].Value : szName + szExt;
string szMatchKey = dictDBResults.Keys.FirstOrDefault(skey => skey.Contains(szThumb));
if (!String.IsNullOrEmpty(szMatchKey)) // leave it in the dictionary - don't remove it - because we may yet hit the Mp4 or the JPG.
{
return;
}
}
else if (fJpg)
{
szThumb = MFBImageInfo.ThumbnailPrefix + szName + szExt;
}
string szPrimary = MFBImageInfo.PrimaryKeyForValues(ic, szKey, szThumb);
// if it is found, super - remove it from the dictionary (for performance) and return
if (dictDBResults.ContainsKey(szPrimary))
{
dictDBResults.Remove(szPrimary);
}
else
{
cOrphansFound++;
cBytesToFree += o.Size;
if (onDelete(o.Key, (int)((100 * cOrphansFound) / cOrphansLikely)))
{
// Make sure that the item
DeleteObjectRequest dor = new DeleteObjectRequest()
{
BucketName = AWSConfiguration.S3BucketName, Key = o.Key
};
s3.DeleteObject(dor);
}
}
});
onSummary?.Invoke(cFilesOnS3, cBytesOnS3, cOrphansFound, cBytesToFree);
}
}
/// <summary>
/// Uploads a video for transcoding. Deletes the source file named in szFileName
/// </summary>
/// <param name="szFileName">Filename of the input stream</param>
/// <param name="szContentType">Content type of the video file</param>
/// <param name="szBucket">Bucket to use. Specified as a parameter because CurrentBucket might return the wrong value when called on a background thread</param>
/// <param name="szPipelineID">PipelineID from amazon</param>
/// <param name="mfbii">The MFBImageInfo that encapsulates the video</param>
public void UploadVideo(string szFileName, string szContentType, string szBucket, string szPipelineID, MFBImageInfo mfbii)
{
if (mfbii == null)
{
throw new ArgumentNullException(nameof(mfbii));
}
using (var etsClient = AWSConfiguration.ElasticTranscoderClient())
{
string szGuid = Guid.NewGuid() + MFBImageInfo.szNewS3KeySuffix;
string szBasePath = mfbii.VirtualPath.StartsWith("/", StringComparison.Ordinal) ? mfbii.VirtualPath.Substring(1) : mfbii.VirtualPath;
string szBaseFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}", szBasePath, szGuid);
PutObjectRequest por = new PutObjectRequest()
{
BucketName = szBucket,
ContentType = szContentType,
AutoCloseStream = true,
InputStream = new FileStream(szFileName, FileMode.Open, FileAccess.Read),
Key = szBaseFile + FileExtensions.VidInProgress,
CannedACL = S3CannedACL.PublicRead,
StorageClass = S3StorageClass.Standard // vs. reduced
};
lock (lockObject)
{
try
{
using (por.InputStream)
{
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
s3.PutObject(por);
File.Delete(szFileName);
}
}
catch (Exception ex)
{
util.NotifyAdminEvent("Error putting video file", ex.Message, ProfileRoles.maskSiteAdminOnly);
throw;
}
}
var job = etsClient.CreateJob(new CreateJobRequest()
{
PipelineId = szPipelineID,
Input = new JobInput()
{
AspectRatio = "auto",
Container = "auto",
FrameRate = "auto",
Interlaced = "auto",
Resolution = "auto",
Key = por.Key,
},
Output = new CreateJobOutput()
{
ThumbnailPattern = String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}{3}", szBasePath, MFBImageInfo.ThumbnailPrefixVideo, szGuid, "{count}"),
Rotate = "auto",
// Generic 720p: Go to http://docs.aws.amazon.com/elastictranscoder/latest/developerguide/create-job.html#PresetId to see a list of some
// of the support presets or call the ListPresets operation to get the full list of available presets
// PresetId = "1351620000000-000010",
PresetId = "1423799228749-hsj7ba",
Key = szBaseFile + FileExtensions.MP4
}
});
if (job != null)
{
new PendingVideo(szGuid, job.Job.Id, mfbii.Comment, mfbii.Class, mfbii.Key, szBucket).Commit();
}
}
}
/// <summary>
/// Moves the image to Amazon
/// </summary>
/// <param name="por"></param>
/// <param name="mfbii">The object to move.</param>
public void MoveByRequest(PutObjectRequest por, MFBImageInfo mfbii)
{
if (mfbii == null)
{
throw new ArgumentNullException(nameof(mfbii));
}
if (por == null)
{
throw new ArgumentNullException(nameof(por));
}
lock (lockObject)
{
try
{
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
PutObjectResponse s3r = null;
using (por.InputStream)
{
s3r = s3.PutObject(por);
}
if (s3r != null)
{
switch (mfbii.ImageType)
{
case MFBImageInfo.ImageFileType.JPEG:
File.Delete(mfbii.PhysicalPathFull);
break;
case MFBImageInfo.ImageFileType.PDF:
{
try
{
if (String.IsNullOrEmpty(mfbii.Comment))
{
mfbii.Comment = mfbii.ThumbnailFile;
}
mfbii.ImageType = MFBImageInfo.ImageFileType.S3PDF;
mfbii.RenameLocalFile(mfbii.ThumbnailFile.Replace(FileExtensions.PDF, FileExtensions.S3PDF));
// Write the comment to the resulting file.
using (FileStream fs = File.OpenWrite(mfbii.PhysicalPathThumbnail))
{
fs.SetLength(0);
byte[] rgBytes = Encoding.UTF8.GetBytes(mfbii.Comment.ToCharArray());
fs.Write(rgBytes, 0, rgBytes.Length);
}
}
catch (Exception ex) when(ex is UnauthorizedAccessException || ex is FileNotFoundException || ex is IOException)
{
mfbii.ImageType = MFBImageInfo.ImageFileType.PDF;
}
}
break;
default:
break;
}
// ALWAYS update the db
mfbii.UpdateDBLocation(false);
}
}
}
catch (AmazonS3Exception ex)
{
throw new MyFlightbookException(WrapAmazonS3Exception(ex), ex);
}
}
}
/// <summary>
/// Creates the thumbnail from a completed video process. Sets the width/height in the process.
/// </summary>
/// <param name="szBasePath">The base path for the video object</param>
/// <param name="szPhysicalPath">The filename to use for the resulting thumbnail image</param>
public void InitThumbnail(string szBasePath, string szPhysicalPath)
{
if (szBasePath == null)
{
throw new ArgumentNullException(nameof(szBasePath));
}
string szThumbFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}00001{2}", MFBImageInfo.ThumbnailPrefixVideo, GUID, FileExtensions.JPG);
if (szBasePath.StartsWith("/", StringComparison.Ordinal))
{
szBasePath = szBasePath.Substring(1);
}
string srcFile = szBasePath + szThumbFile;
// Copy the thumbnail over
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
try
{
using (GetObjectResponse gor = s3.GetObject(new GetObjectRequest()
{
BucketName = Bucket, Key = srcFile
}))
{
if (gor != null && gor.ResponseStream != null)
{
#pragma warning disable IDISP007 // Don't dispose injected
// Amazon documents EXPLICITLY say we should wrap in a using block. See https://docs.aws.amazon.com/sdkfornet1/latest/apidocs/html/P_Amazon_S3_Model_S3Response_ResponseStream.htm
using (gor.ResponseStream)
#pragma warning restore IDISP007 // Don't dispose injected.
{
using (System.Drawing.Image image = MFBImageInfo.DrawingCompatibleImageFromStream(gor.ResponseStream))
{
Info inf = MFBImageInfo.InfoFromImage(image);
// save the thumbnail locally.
inf.ImageDescription = Comment;
Bitmap bmp = MFBImageInfo.BitmapFromImage(inf.Image, MFBImageInfo.ThumbnailHeight, MFBImageInfo.ThumbnailWidth);
ThumbWidth = bmp.Width;
ThumbHeight = bmp.Height;
using (bmp)
{
// get all properties of the original image and copy them to the new image. This should include the annotation (above)
foreach (PropertyItem pi in inf.Image.PropertyItems)
{
bmp.SetPropertyItem(pi);
}
bmp.Save(szPhysicalPath, ImageFormat.Jpeg);
}
}
}
}
}
}
catch (AmazonS3Exception)
{
// Thumbnail was not found - audio file perhaps? Use the generic audio file.
System.IO.File.Copy(HostingEnvironment.MapPath("~/images/audio.png"), szPhysicalPath);
}
// clean up the folder on S3 - anything that has the GUID but not .mp4 in it or the thumbnail in it. (Save space!) i.e., delete excess thumbnails and the source video file.
List <S3Object> lstS3Objects = new List <S3Object>();
ListObjectsRequest loRequest = new ListObjectsRequest()
{
BucketName = Bucket, Prefix = szBasePath
};
// Get the list of S3 objects
do
{
ListObjectsResponse response = s3.ListObjects(loRequest);
foreach (S3Object o in response.S3Objects)
{
if (o.Key.Contains(GUID) && !o.Key.Contains(FileExtensions.MP4) && !o.Key.Contains(szThumbFile))
{
lstS3Objects.Add(o);
}
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
loRequest.Marker = response.NextMarker;
}
else
{
loRequest = null;
}
} while (loRequest != null);
lstS3Objects.ForEach((o) =>
{
s3.DeleteObject(new DeleteObjectRequest()
{
BucketName = Bucket, Key = o.Key
});
});
}
}
/// <summary>
/// Creates the thumbnail from a completed video process. Sets the width/height in the process.
/// </summary>
/// <param name="szBasePath">The base path for the video object</param>
/// <param name="szPhysicalPath">The filename to use for the resulting thumbnail image</param>
public void InitThumbnail(string szBasePath, string szPhysicalPath)
{
if (szBasePath == null)
{
throw new ArgumentNullException(nameof(szBasePath));
}
string szThumbFile = String.Format(CultureInfo.InvariantCulture, "{0}{1}00001{2}", MFBImageInfo.ThumbnailPrefixVideo, GUID, FileExtensions.JPG);
if (szBasePath.StartsWith("/", StringComparison.Ordinal))
{
szBasePath = szBasePath.Substring(1);
}
string srcFile = szBasePath + szThumbFile;
// Copy the thumbnail over
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
GetObjectResponse gor = s3.GetObject(new GetObjectRequest()
{
BucketName = Bucket, Key = srcFile
});
if (gor != null && gor.ResponseStream != null)
{
using (gor.ResponseStream)
{
using (System.Drawing.Image image = MFBImageInfo.DrawingCompatibleImageFromStream(gor.ResponseStream))
{
Info inf = MFBImageInfo.InfoFromImage(image);
// save the thumbnail locally.
using (inf.Image)
{
inf.ImageDescription = Comment;
Bitmap bmp = MFBImageInfo.BitmapFromImage(inf.Image, MFBImageInfo.ThumbnailHeight, MFBImageInfo.ThumbnailWidth);
ThumbWidth = bmp.Width;
ThumbHeight = bmp.Height;
using (bmp)
{
// get all properties of the original image and copy them to the new image. This should include the annotation (above)
foreach (PropertyItem pi in inf.Image.PropertyItems)
{
bmp.SetPropertyItem(pi);
}
bmp.Save(szPhysicalPath, ImageFormat.Jpeg);
}
}
}
}
}
// clean up the folder on S3 - anything that has the GUID but not .mp4 in it or the thumbnail in it. (Save space!) i.e., delete excess thumbnails and the source video file.
List <S3Object> lstS3Objects = new List <S3Object>();
ListObjectsRequest loRequest = new ListObjectsRequest()
{
BucketName = Bucket, Prefix = szBasePath
};
// Get the list of S3 objects
do
{
ListObjectsResponse response = s3.ListObjects(loRequest);
foreach (S3Object o in response.S3Objects)
{
if (o.Key.Contains(GUID) && !o.Key.Contains(FileExtensions.MP4) && !o.Key.Contains(szThumbFile))
{
lstS3Objects.Add(o);
}
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
loRequest.Marker = response.NextMarker;
}
else
{
loRequest = null;
}
} while (loRequest != null);
lstS3Objects.ForEach((o) =>
{
s3.DeleteObject(new DeleteObjectRequest()
{
BucketName = Bucket, Key = o.Key
});
});
}
}
