/// <summary>
/// ADMIN ONLY - Remove images from S3 LIVE BUCKET that are orphaned (no reference from live site)
/// </summary>
/// <param name="ic"></param>
/// <param name="handleS3Object"></param>
public static void ADMINDeleteS3Orphans(MFBImageInfo.ImageClass ic, Action <long, long, long, long> onSummary, Action onS3EnumDone, Func <string, int, bool> onDelete)
{
string szBasePath = MFBImageInfo.BasePathFromClass(ic);
if (szBasePath.StartsWith("/", StringComparison.InvariantCultureIgnoreCase))
{
szBasePath = szBasePath.Substring(1);
}
Regex r = new Regex(String.Format(CultureInfo.InvariantCulture, "{0}(.*)/(.*)(\\.jpg|\\.jpeg|\\.pdf|\\.s3pdf|\\.mp4)", szBasePath), RegexOptions.IgnoreCase);
long cBytesToFree = 0;
long cBytesOnS3 = 0;
long cFilesOnS3 = 0;
long cOrphansFound = 0;
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = AWSConfiguration.S3BucketName, Prefix = szBasePath
};
/*
* Need to get the files from S3 FIRST, otherwise there is a race condition
* I.e., if we get files from the DB, then get files from S3, a file could be added to the db AFTER our query
* but BEFORE we retrieve it from the S3 listing, and we will thus treat it as an orphan and delete it.
* But since the file is put into the DB before being moved to S3, if we get all the S3 files and THEN
* get the DB references, we will always have a subset of the valid S3 files, which prevents a false positive
* orphan identification.
*/
List <S3Object> lstS3Objects = new List <S3Object>();
// Get the list of S3 objects
do
{
ListObjectsResponse response = s3.ListObjects(request);
cFilesOnS3 += response.S3Objects.Count;
foreach (S3Object o in response.S3Objects)
{
cBytesOnS3 += o.Size;
lstS3Objects.Add(o);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
onS3EnumDone?.Invoke();
// Now get all of the images in the class and do orphan detection
Dictionary <string, MFBImageInfo> dictDBResults = MFBImageInfo.AllImagesForClass(ic);
long cOrphansLikely = Math.Max(lstS3Objects.Count - dictDBResults.Keys.Count, 1);
Regex rGuid = new Regex(String.Format(CultureInfo.InvariantCulture, "^({0})?([^_]*).*", MFBImageInfo.ThumbnailPrefixVideo), RegexOptions.Compiled); // for use below in extracting GUIDs from video thumbnail and video file names.
lstS3Objects.ForEach((o) =>
{
Match m = r.Match(o.Key);
if (m.Groups.Count < 3)
{
return;
}
string szKey = m.Groups[1].Value;
string szName = m.Groups[2].Value;
string szExt = m.Groups[3].Value;
bool fPDF = (String.Compare(szExt, FileExtensions.PDF, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVid = (String.Compare(szExt, FileExtensions.MP4, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVidThumb = Regex.IsMatch(szExt, FileExtensions.RegExpImageFileExtensions) && szName.StartsWith(MFBImageInfo.ThumbnailPrefixVideo, StringComparison.InvariantCultureIgnoreCase);
bool fJpg = (String.Compare(szExt, FileExtensions.JPG, StringComparison.InvariantCultureIgnoreCase) == 0 || String.Compare(szExt, FileExtensions.JPEG, StringComparison.InvariantCultureIgnoreCase) == 0);
string szThumb = string.Empty;
if (fPDF)
{
szThumb = szName + FileExtensions.S3PDF;
}
else if (fVid || fVidThumb)
{
// This is a bit of a hack, but we have two files on the server for a video, neither of which precisely matches what's in the database.
// The video file is {guid}.mp4 or {guid}_.mp4, the thumbnail on S3 is v_{guid}_0001.jpg.
// So we grab the GUID and see if we have a database entry matching that guid.
Match mGuid = rGuid.Match(szName);
szThumb = (mGuid.Groups.Count >= 3) ? mGuid.Groups[2].Value : szName + szExt;
string szMatchKey = dictDBResults.Keys.FirstOrDefault(skey => skey.Contains(szThumb));
if (!String.IsNullOrEmpty(szMatchKey)) // leave it in the dictionary - don't remove it - because we may yet hit the Mp4 or the JPG.
{
return;
}
}
else if (fJpg)
{
szThumb = MFBImageInfo.ThumbnailPrefix + szName + szExt;
}
string szPrimary = MFBImageInfo.PrimaryKeyForValues(ic, szKey, szThumb);
// if it is found, super - remove it from the dictionary (for performance) and return
if (dictDBResults.ContainsKey(szPrimary))
{
dictDBResults.Remove(szPrimary);
}
else
{
cOrphansFound++;
cBytesToFree += o.Size;
if (onDelete(o.Key, (int)((100 * cOrphansFound) / cOrphansLikely)))
{
// Make sure that the item
DeleteObjectRequest dor = new DeleteObjectRequest()
{
BucketName = AWSConfiguration.S3BucketName, Key = o.Key
};
s3.DeleteObject(dor);
}
}
});
onSummary?.Invoke(cFilesOnS3, cBytesOnS3, cOrphansFound, cBytesToFree);
}
}
protected void SyncImages(List <DirKey> lstDk)
{
if (lstDk == null)
{
throw new ArgumentNullException(nameof(lstDk));
}
Response.Clear();
Response.Write(String.Format(CultureInfo.InvariantCulture, "<html><head><link href=\"{0}\" rel=\"stylesheet\" type=\"text/css\" /></head><body><p>", VirtualPathUtility.ToAbsolute("~/Public/stylesheet.css")));
DateTime dtStart = DateTime.Now;
int cNewImages = 0;
int cChangedImages = 0;
int cDeletedImages = 0;
int cImagesExamined = 0;
List <MFBImageInfo> lstMfbiiNewOrChanged = new List <MFBImageInfo>();
try
{
// Get a list of all of the images in the DB for this category:
UpdateProgress(1, 0, String.Format(CultureInfo.CurrentCulture, "Getting images for {0} from DB", CurrentSource.ToString()));
Dictionary <string, MFBImageInfo> dictDBResults = MFBImageInfo.AllImagesForClass(CurrentSource);
int cDBEntriesToStart = dictDBResults.Count;
// Get a list of all of the images in this category:
int cDirectories = lstDk.Count;
int cDirectoriesProcessed = 0;
int c5Percent = Math.Max(1, cDirectories / 20);
UpdateProgress(2, 0, String.Format(CultureInfo.CurrentCulture, "Getting images for {0} ({1} folders)...", CurrentSource.ToString(), cDirectories));
ImageList il = new ImageList
{
Class = CurrentSource
};
foreach (DirKey dk in lstDk)
{
il.Key = dk.Key;
il.Refresh(true);
foreach (MFBImageInfo mfbii in il.ImageArray)
{
MFBImageInfo mfbiiMatch = dictDBResults.ContainsKey(mfbii.PrimaryKey) ? dictDBResults[mfbii.PrimaryKey] : null;
if (mfbiiMatch == null)
{
// If no match was found, it's a new image
lstMfbiiNewOrChanged.Add(mfbii);
cNewImages++;
}
else
{
bool fCommentChanged = String.Compare(mfbii.Comment, mfbiiMatch.Comment, StringComparison.CurrentCultureIgnoreCase) != 0;
bool fLocChanged = !MyFlightbook.Geography.LatLong.AreEqual(mfbii.Location, mfbiiMatch.Location);
// if it's changed, we need to update it.
if (fCommentChanged || fLocChanged)
{
if (mfbii.Location != null && !mfbii.Location.IsValid)
{
mfbii.Location = null;
}
UpdateProgress(2, (100 * cDirectoriesProcessed) / cDirectories, String.Format(CultureInfo.CurrentCulture, "Changed {0}:{1}{2}",
mfbii.PrimaryKey,
fCommentChanged ? String.Format(CultureInfo.CurrentCulture, "<br /> Comment: {0} => {1}", mfbiiMatch.Comment, mfbii.Comment) : string.Empty,
fLocChanged ? String.Format(CultureInfo.CurrentCulture, "<br /> Location: {0} => {1}", (mfbiiMatch.Location == null ? "null" : mfbiiMatch.Location.ToString()), (mfbii.Location == null ? "null" : mfbii.Location.ToString())) : string.Empty));
lstMfbiiNewOrChanged.Add(mfbii);
cChangedImages++;
}
// Now remove it from the list of DB images.
dictDBResults.Remove(mfbii.PrimaryKey);
mfbiiMatch.UnCache();
mfbiiMatch = null; // save some memory?
}
mfbii.UnCache();
cImagesExamined++;
}
il.Clear();
if (++cDirectoriesProcessed % c5Percent == 0)
{
GC.Collect();
UpdateProgress(2, (100 * cDirectoriesProcessed) / cDirectories, String.Format(CultureInfo.CurrentCulture, "{0} Folders processed, {1} images found", cDirectoriesProcessed, cImagesExamined));
}
}
UpdateProgress(3, 100, String.Format(CultureInfo.CurrentCulture, "Elapsed Time: {0} seconds", DateTime.Now.Subtract(dtStart).TotalSeconds));
UpdateProgress(4, 0, String.Format(CultureInfo.CurrentCulture, "{0} image files found, {1} images in DB", cImagesExamined, cDBEntriesToStart));
// Now see if anything got deleted but not from the DB
int cImagesToDelete = dictDBResults.Values.Count;
if (String.IsNullOrEmpty(Request["preview"]))
{
UpdateProgress(5, 0, String.Format(CultureInfo.CurrentCulture, "{0} images found in DB that weren't found as files; deleting these.", cImagesToDelete));
foreach (MFBImageInfo mfbii in dictDBResults.Values)
{
mfbii.DeleteFromDB();
cDeletedImages++;
if (cDeletedImages % 100 == 0)
{
UpdateProgress(5, (100 * cDeletedImages) / cImagesToDelete, String.Format(CultureInfo.CurrentCulture, "Deleted {0} images from DB", cDeletedImages));
}
}
// And finally add new images back
UpdateProgress(6, 0, String.Format(CultureInfo.CurrentCulture, "{0} new and {1} changed images (total of {2}) to be added to/updated in the DB", cNewImages, cChangedImages, lstMfbiiNewOrChanged.Count));
lstMfbiiNewOrChanged.ForEach((mfbii) =>
{
try
{
mfbii.ToDB();
}
catch (Exception ex) when(ex is MyFlightbookException)
{
Response.Write(String.Format(CultureInfo.InvariantCulture, "<p class=\"error\">Exception sync'ing DB: {0}</p>", ex.Message));
}
});
}
else
{
UpdateProgress(5, 0, String.Format(CultureInfo.CurrentCulture, "{0} images found in DB that weren't found as files; these are:", cImagesToDelete));
foreach (MFBImageInfo mfbii in dictDBResults.Values)
{
UpdateProgress(5, 0, mfbii.PathThumbnail);
}
UpdateProgress(6, 0, String.Format(CultureInfo.CurrentCulture, "{0} images found that are new or changed that weren't in DB; these are:", cNewImages + cChangedImages));
lstMfbiiNewOrChanged.ForEach((mfbii) => { UpdateProgress(6, 0, mfbii.PathThumbnail); });
}
UpdateProgress(6, 100, "Finished!");
}
catch (Exception ex) when(!(ex is OutOfMemoryException))
{
Response.Write(String.Format(CultureInfo.InvariantCulture, "<p class=\"error\">Exception sync'ing DB: {0}</p>", ex.Message));
}
Response.Write("</p></body></html>");
Response.End();
}
