protected void Page_Load(object sender, EventArgs e)
{
Master.SelectedTab = tabID.tabAdmin;
CheckAdmin(Profile.GetUser(Page.User.Identity.Name).CanManageData);
bool fNumericKeySort = true;
string szRoot = util.GetStringParam(Request, "r");
if (!Enum.TryParse <MFBImageInfo.ImageClass>(szRoot, out MFBImageInfo.ImageClass imageClass))
{
imageClass = MFBImageInfo.ImageClass.Flight;
}
CurrentSource = imageClass;
switch (CurrentSource)
{
case MFBImageInfo.ImageClass.Flight:
m_szLinkTemplate = "~/member/logbookNew.aspx/{0}?a=1";
break;
case MFBImageInfo.ImageClass.Aircraft:
m_szLinkTemplate = "~/member/EditAircraft.aspx?a=1&id={0}";
break;
case MFBImageInfo.ImageClass.Endorsement:
case MFBImageInfo.ImageClass.OfflineEndorsement:
fNumericKeySort = false;
break;
case MFBImageInfo.ImageClass.BasicMed:
break;
}
szBase = MFBImageInfo.BasePathFromClass(CurrentSource);
bool fIsSync = (util.GetIntParam(Request, "sync", 0) != 0);
bool fIsS3Orphan = (util.GetIntParam(Request, "dels3orphan", 0) != 0);
int cAutoMigrate = util.GetIntParam(Request, "automigrate", 0);
List <DirKey> lstDk = null;
if (!IsPostBack)
{
if (fIsSync)
{
lstDk = new List <DirKey>();
if (!String.IsNullOrEmpty(szBase))
{
DirectoryInfo dir = new DirectoryInfo(System.Web.Hosting.HostingEnvironment.MapPath(szBase));
DirectoryInfo[] rgSubDir = dir.GetDirectories();
int i = 0;
foreach (DirectoryInfo di in rgSubDir)
{
// Delete the directory if it is empty
FileInfo[] rgfi = di.GetFiles();
DirectoryInfo[] rgdi = di.GetDirectories();
if (rgfi.Length == 0 && rgdi.Length == 0)
{
di.Delete();
continue;
}
DirKey dk = new DirKey
{
Key = di.Name
};
if (fNumericKeySort)
{
if (Int32.TryParse(dk.Key, out int num))
{
dk.SortID = num;
}
else
{
dk.SortID = i;
}
}
else
{
dk.SortID = i;
}
i++;
lstDk.Add(dk);
}
}
lstDk.Sort();
}
else if (cAutoMigrate == 0)
{
// Get the total # of image rows
DBHelper dbh = new DBHelper(String.Format(CultureInfo.InvariantCulture, "SELECT COUNT(DISTINCT imagekey) AS NumRows FROM images WHERE virtpathid={0}", (int)CurrentSource));
dbh.ReadRow((comm) => { }, (dr) => { TotalImageRows = Convert.ToInt32(dr["NumRows"], CultureInfo.InvariantCulture); });
CurrentImageRowOffset = 0;
}
}
if (fIsSync)
{
SyncImages(lstDk);
}
else if (fIsS3Orphan)
{
DeleteS3Orphans();
}
else if (cAutoMigrate != 0)
{
txtLimitFiles.Text = Math.Min(Math.Max(cAutoMigrate, 10), 100).ToString(CultureInfo.CurrentCulture);
btnMigrateImages_Click(sender, e);
}
else
{
UpdateGrid();
}
}
/// <summary>
/// ADMIN ONLY - Remove images from S3 LIVE BUCKET that are orphaned (no reference from live site)
/// </summary>
/// <param name="ic"></param>
/// <param name="handleS3Object"></param>
public static void ADMINDeleteS3Orphans(MFBImageInfo.ImageClass ic, Action <long, long, long, long> onSummary, Action onS3EnumDone, Func <string, int, bool> onDelete)
{
string szBasePath = MFBImageInfo.BasePathFromClass(ic);
if (szBasePath.StartsWith("/", StringComparison.InvariantCultureIgnoreCase))
{
szBasePath = szBasePath.Substring(1);
}
Regex r = new Regex(String.Format(CultureInfo.InvariantCulture, "{0}(.*)/(.*)(\\.jpg|\\.jpeg|\\.pdf|\\.s3pdf|\\.mp4)", szBasePath), RegexOptions.IgnoreCase);
long cBytesToFree = 0;
long cBytesOnS3 = 0;
long cFilesOnS3 = 0;
long cOrphansFound = 0;
using (IAmazonS3 s3 = AWSConfiguration.S3Client())
{
ListObjectsRequest request = new ListObjectsRequest()
{
BucketName = AWSConfiguration.S3BucketName, Prefix = szBasePath
};
/*
* Need to get the files from S3 FIRST, otherwise there is a race condition
* I.e., if we get files from the DB, then get files from S3, a file could be added to the db AFTER our query
* but BEFORE we retrieve it from the S3 listing, and we will thus treat it as an orphan and delete it.
* But since the file is put into the DB before being moved to S3, if we get all the S3 files and THEN
* get the DB references, we will always have a subset of the valid S3 files, which prevents a false positive
* orphan identification.
*/
List <S3Object> lstS3Objects = new List <S3Object>();
// Get the list of S3 objects
do
{
ListObjectsResponse response = s3.ListObjects(request);
cFilesOnS3 += response.S3Objects.Count;
foreach (S3Object o in response.S3Objects)
{
cBytesOnS3 += o.Size;
lstS3Objects.Add(o);
}
// If response is truncated, set the marker to get the next
// set of keys.
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
onS3EnumDone?.Invoke();
// Now get all of the images in the class and do orphan detection
Dictionary <string, MFBImageInfo> dictDBResults = MFBImageInfo.AllImagesForClass(ic);
long cOrphansLikely = Math.Max(lstS3Objects.Count - dictDBResults.Keys.Count, 1);
Regex rGuid = new Regex(String.Format(CultureInfo.InvariantCulture, "^({0})?([^_]*).*", MFBImageInfo.ThumbnailPrefixVideo), RegexOptions.Compiled); // for use below in extracting GUIDs from video thumbnail and video file names.
lstS3Objects.ForEach((o) =>
{
Match m = r.Match(o.Key);
if (m.Groups.Count < 3)
{
return;
}
string szKey = m.Groups[1].Value;
string szName = m.Groups[2].Value;
string szExt = m.Groups[3].Value;
bool fPDF = (String.Compare(szExt, FileExtensions.PDF, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVid = (String.Compare(szExt, FileExtensions.MP4, StringComparison.InvariantCultureIgnoreCase) == 0);
bool fVidThumb = Regex.IsMatch(szExt, FileExtensions.RegExpImageFileExtensions) && szName.StartsWith(MFBImageInfo.ThumbnailPrefixVideo, StringComparison.InvariantCultureIgnoreCase);
bool fJpg = (String.Compare(szExt, FileExtensions.JPG, StringComparison.InvariantCultureIgnoreCase) == 0 || String.Compare(szExt, FileExtensions.JPEG, StringComparison.InvariantCultureIgnoreCase) == 0);
string szThumb = string.Empty;
if (fPDF)
{
szThumb = szName + FileExtensions.S3PDF;
}
else if (fVid || fVidThumb)
{
// This is a bit of a hack, but we have two files on the server for a video, neither of which precisely matches what's in the database.
// The video file is {guid}.mp4 or {guid}_.mp4, the thumbnail on S3 is v_{guid}_0001.jpg.
// So we grab the GUID and see if we have a database entry matching that guid.
Match mGuid = rGuid.Match(szName);
szThumb = (mGuid.Groups.Count >= 3) ? mGuid.Groups[2].Value : szName + szExt;
string szMatchKey = dictDBResults.Keys.FirstOrDefault(skey => skey.Contains(szThumb));
if (!String.IsNullOrEmpty(szMatchKey)) // leave it in the dictionary - don't remove it - because we may yet hit the Mp4 or the JPG.
{
return;
}
}
else if (fJpg)
{
szThumb = MFBImageInfo.ThumbnailPrefix + szName + szExt;
}
string szPrimary = MFBImageInfo.PrimaryKeyForValues(ic, szKey, szThumb);
// if it is found, super - remove it from the dictionary (for performance) and return
if (dictDBResults.ContainsKey(szPrimary))
{
dictDBResults.Remove(szPrimary);
}
else
{
cOrphansFound++;
cBytesToFree += o.Size;
if (onDelete(o.Key, (int)((100 * cOrphansFound) / cOrphansLikely)))
{
// Make sure that the item
DeleteObjectRequest dor = new DeleteObjectRequest()
{
BucketName = AWSConfiguration.S3BucketName, Key = o.Key
};
s3.DeleteObject(dor);
}
}
});
onSummary?.Invoke(cFilesOnS3, cBytesOnS3, cOrphansFound, cBytesToFree);
}
}
protected void Page_Load(object sender, EventArgs e)
{
Master.SelectedTab = tabID.tabAdmin;
if (!MyFlightbook.Profile.GetUser(Page.User.Identity.Name).CanManageData)
{
util.NotifyAdminEvent("Attempt to view admin page", String.Format("User {0} tried to hit the admin page.", Page.User.Identity.Name), ProfileRoles.maskSiteAdminOnly);
Response.Redirect("~/HTTP403.htm");
}
bool fNumericKeySort = true;
string szRoot = util.GetStringParam(Request, "r");
CurrentSource = MFBImageInfo.ImageClass.Flight;
try
{
CurrentSource = (MFBImageInfo.ImageClass)Enum.Parse(typeof(MFBImageInfo.ImageClass), szRoot);
}
catch { }
switch (CurrentSource)
{
case MFBImageInfo.ImageClass.Flight:
m_szLinkTemplate = "~/member/logbookNew.aspx/{0}?a=1";
break;
case MFBImageInfo.ImageClass.Aircraft:
m_szLinkTemplate = "~/member/EditAircraft.aspx?a=1&id={0}";
break;
case MFBImageInfo.ImageClass.Endorsement:
fNumericKeySort = false;
break;
case MFBImageInfo.ImageClass.BasicMed:
break;
}
szBase = MFBImageInfo.BasePathFromClass(CurrentSource);
bool fIsSync = (util.GetIntParam(Request, "sync", 0) != 0);
bool fIsS3Orphan = (util.GetIntParam(Request, "dels3orphan", 0) != 0);
int cAutoMigrate = util.GetIntParam(Request, "automigrate", 0);
List <DirKey> lstDk = null;
if (!IsPostBack)
{
if (fIsSync)
{
lstDk = new List <DirKey>();
if (!String.IsNullOrEmpty(szBase))
{
DirectoryInfo dir = new DirectoryInfo(System.Web.Hosting.HostingEnvironment.MapPath(szBase));
DirectoryInfo[] rgSubDir = dir.GetDirectories();
int i = 0;
foreach (DirectoryInfo di in rgSubDir)
{
// Delete the directory if it is empty
FileInfo[] rgfi = di.GetFiles();
DirectoryInfo[] rgdi = di.GetDirectories();
if (rgfi.Length == 0 && rgdi.Length == 0)
{
di.Delete();
continue;
}
DirKey dk = new DirKey();
dk.Key = di.Name;
if (fNumericKeySort)
{
try
{
dk.SortID = Convert.ToInt32(dk.Key);
}
catch
{
dk.SortID = i;
}
}
else
{
dk.SortID = i;
}
i++;
lstDk.Add(dk);
}
}
lstDk.Sort();
}
else if (cAutoMigrate == 0)
{
// Get the total # of image rows
DBHelper dbh = new DBHelper(String.Format("SELECT COUNT(DISTINCT imagekey) AS NumRows FROM images WHERE virtpathid={0}", (int)CurrentSource));
dbh.ReadRow((comm) => { }, (dr) => { TotalImageRows = Convert.ToInt32(dr["NumRows"]); });
CurrentImageRowOffset = 0;
}
}
if (fIsSync)
{
SyncImages(lstDk);
}
else if (fIsS3Orphan)
{
DeleteS3Orphans();
}
else if (cAutoMigrate != 0)
{
txtLimitFiles.Text = Math.Min(Math.Max(cAutoMigrate, 10), 100).ToString();
btnMigrateImages_Click(sender, e);
}
else
{
UpdateGrid();
}
}
