public RemoteFileInfo[] GetFileListing(string folder, DirectoryListOptions listOptions)
{
//MAKE SURE THE FOLDER IS IN THE PROPER FORMAT FOR THE DATABASE
folder = FormatFolder(folder);
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray().Shuffle()[0].ConnectionString))
{
if (listOptions == DirectoryListOptions.SelectedFolderOnly)
{
return(tmpDB.FileMarkers.Where(a => a.Folder == folder).Select(a => new RemoteFileInfo()
{
Folder = a.Folder, FileName = a.FileName, LastWriteTime = a.LastWriteTime, LastModTime = a.LastModTime, LastReadTime = a.LastReadTime, Length = a.Length, RemoteLocations = a.FileLocations.Select(b => b.Location).ToArray()
}).ToArray());
}
else if (listOptions == DirectoryListOptions.AllSubDirectories)
{
return(tmpDB.FileMarkers.Where(a => a.Folder.StartsWith(folder)).Select(a => new RemoteFileInfo()
{
Folder = a.Folder, FileName = a.FileName, LastWriteTime = a.LastWriteTime, LastModTime = a.LastModTime, LastReadTime = a.LastReadTime, Length = a.Length, RemoteLocations = a.FileLocations.Select(b => b.Location).ToArray()
}).ToArray());
}
else
{
throw new ArgumentException("Invalid parameter specified for listingOptions");
}
}
}
public string BeginGetFileChunk(string folder, string fileName)
{
string tmpFileID = Guid.NewGuid().ToString();
string tmpFileName = Path.Combine(serverSettings.TempFolder, tmpFileID);
//USE A BASIC WEBCLIENT TO ATTEMPT TO DOWNLOAD THE FILE FROM THE STORAGE LOCATION
WebClient tmpClient = new WebClient();
//MAKE SURE THE FOLDER IS IN THE PROPER FORMAT FOR THE DATABASE
folder = FormatFolder(folder);
//USED THE FIRST CONNECTION IN THE DATABASE WHILE PREFORMING READS IT IS CONSIDERED THE "MASTER"
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray().Shuffle()[0].ConnectionString))
{
FileMarker tmpMarker = tmpDB.FileMarkers.Where(a => a.Folder == folder && a.FileName == fileName).FirstOrDefault();
//IF THE DATABASE DID NOT HAVE THIS FILE THEN RETURN AN ERROR BECAUSE FILE WAS NOT FOUND
if (tmpMarker != null)
{
//FOR ATTEMPT TO DOWNLOAD THE FILE FROM EACH LOCATION "SHUFFLED" TO HELP LOAD BALANCE THE SERVERS
foreach (FileLocation tmpLocation in tmpMarker.FileLocations.ToArray().Shuffle())
{
try
{
//DOWNLOAD THE FILE TO A TEMPORARY STORING LOCATION WHERE WE CAN BEGIN TO RECEIVE THE FILE CHUNKS FROM
tmpClient.DownloadFile(string.Format("{0}GetFile.aspx?folder={1}&filename={2}", tmpLocation.Location, Server.UrlEncode(folder), Server.UrlEncode(fileName)), tmpFileName);
//IF THE FILES HASH IS CORRECT THEN RETURN TO CALLING FUNCTION, ELSE DELETE THE TEMP FILE AND TRY AGAIN
if (File.OpenRead(tmpFileName).ComputeMD5(true) == tmpMarker.Hash)
{
File.WriteAllText(tmpFileName + "._placement_", "0");
return(tmpFileID);
}
else
{
Logging.WriteEntry(string.Format("Incorrect Hash Value on file {0}{1}", folder, fileName), null, null);
File.Delete(tmpFileName);
}
}
catch (Exception e)
{
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
}
}
else
{
//FILE WAS NOT FOUND SO THROW AN ERROR INSTEAD OF RETURNING
throw new Exception("File not found!");
}
}
//IF WE HAVE MADE IT THIS FAR THEN THE FILE SHOULD HAVE BEEN FOUND BUT WE HAVE A PROBLEM
throw new Exception("Unable to read file please verify storage integrity!");
}
public bool DeleteFile(string folder, string fileName)
{
//USE A BASIC WEBCLIENT TO ATTEMPT TO DOWNLOAD THE FILE FROM THE STORAGE LOCATION
WebClient tmpClient = new WebClient();
//MAKE SURE THE FOLDER IS IN THE PROPER FORMAT FOR THE DATABASE
folder = FormatFolder(folder);
//USED THE FIRST CONNECTION IN THE DATABASE WHILE PREFORMING READS IT IS CONSIDERED THE "MASTER"
foreach (ConnectionStringSettings tmpConnection in ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray())
{
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(tmpConnection.ConnectionString))
{
FileMarker tmpMarker = tmpDB.FileMarkers.Where(a => a.Folder == folder && a.FileName == fileName).FirstOrDefault();
//IF THE DATABASE DID NOT HAVE THIS FILE THEN RETURN AN ERROR BECAUSE FILE WAS NOT FOUND
if (tmpMarker != null)
{
//FOR ATTEMPT TO DELETE THE FILE FROM EACH OF ITS REMOTE STORAGE LOCATIONS
foreach (FileLocation tmpLocation in tmpMarker.FileLocations.ToArray())
{
try
{
//DELETE THE FILE FROM THE REMOTE STORAGE LOCATION
tmpClient.DownloadData(string.Format("{0}DeleteFile.aspx?folder={1}&filename={2}", tmpLocation.Location, Server.UrlEncode(folder), Server.UrlEncode(fileName)));
//REMOVE THE LOCATION FROM THE DB
tmpDB.FileLocations.DeleteOnSubmit(tmpLocation);
}
catch (Exception e)
{
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
}
//REMOVE THE MARKER FROM THE DB
tmpDB.FileMarkers.DeleteOnSubmit(tmpMarker);
//SUBMIT ALL THE CHANGES TO THE DATABASE FOR UPDATING
tmpDB.SubmitChanges();
}
else
{
//FILE WAS NOT FOUND SO THROW AN ERROR INSTEAD OF RETURNING
throw new Exception("File not found!");
}
}
}
//RETURN SUCCESSFULL FOR THE DELETE
return(true);
}
public byte[] GetFile(string folder, string fileName)
{
//USE A BASIC WEBCLIENT TO ATTEMPT TO DOWNLOAD THE FILE FROM THE STORAGE LOCATION
WebClient tmpClient = new WebClient();
//MAKE SURE THE FOLDER IS IN THE PROPER FORMAT FOR THE DATABASE
folder = FormatFolder(folder);
//USED THE FIRST CONNECTION IN THE DATABASE WHILE PREFORMING READS IT IS CONSIDERED THE "MASTER"
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray().Shuffle()[0].ConnectionString))
{
FileMarker tmpMarker = tmpDB.FileMarkers.Where(a => a.Folder == folder && a.FileName == fileName).FirstOrDefault();
//IF THE DATABASE DID NOT HAVE THIS FILE THEN RETURN AN ERROR BECAUSE FILE WAS NOT FOUND
if (tmpMarker != null)
{
//FOR ATTEMPT TO DOWNLOAD THE FILE FROM EACH LOCATION "SHUFFLED" TO HELP LOAD BALANCE THE SERVERS
foreach (FileLocation tmpLocation in tmpMarker.FileLocations.ToArray().Shuffle())
{
try
{
byte[] result = tmpClient.DownloadData(string.Format("{0}GetFile.aspx?folder={1}&filename={2}", tmpLocation.Location, Server.UrlEncode(folder), Server.UrlEncode(fileName)));
//IF THIS FILE MATCHES WHAT THE HASH SAYS IT SHOULD THEN RETURN IT. OTHERWISE ASSUME ITS CORRUPT
if (BitConverter.ToString(MD5.Create().ComputeHash(result)).Replace("-", "") == tmpMarker.Hash)
{
return(result);
}
else
{
Logging.WriteEntry(string.Format("Incorrect Hash for File: {0}{1} on RemoteStorage: {2}", folder, fileName, tmpLocation.Location), null, null);
}
}
catch (Exception e)
{
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
}
}
else
{
//FILE WAS NOT FOUND SO THROW AN ERROR INSTEAD OF RETURNING
throw new Exception("File not found!");
}
}
//IF WE HAVE MADE IT THIS FAR THEN THE FILE SHOULD HAVE BEEN FOUND BUT WE HAVE A PROBLEM
throw new Exception("Unable to read file please verify storage integrity!");
}
public string[] GetDirectoryListing(string folder, DirectoryListOptions listOptions)
{
//MAKE SURE THE FOLDER IS IN THE PROPER FORMAT FOR THE DATABASE
folder = FormatFolder(folder);
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray().Shuffle()[0].ConnectionString))
{
int folderDepth = folder.Split('/').Length;
if (listOptions == DirectoryListOptions.SelectedFolderOnly)
{
return(tmpDB.FileMarkers.Where(a => a.Folder.StartsWith(folder)).Select(a => a.Folder).Distinct().ToArray().Where(a => a.Split('/').Length == folderDepth + 1).ToArray());
}
else if (listOptions == DirectoryListOptions.AllSubDirectories)
{
return(tmpDB.FileMarkers.Where(a => a.Folder.StartsWith(folder)).Select(a => a.Folder).Distinct().ToArray());
}
else
{
throw new ArgumentException("Invalid parameter specified for listingOptions");
}
}
}
/// <summary>
/// DISTRIBUTES THE FILES AMONGST THE REMOTE STORAGE LOCATIONS
/// </summary>
/// <param name="folder">FOLDER WHERE FILE IS TO BE STORED</param>
/// <param name="fileName">FILENAME OF THE FILE BEING STORED</param>
/// <param name="replicateCount">NUMBERS OF TIMES TO REPLICATE THE FILE NOT INCLUDING THE INITIAL</param>
/// <param name="tmpFileName">LOCATION OF THE LOCAL FILE TO UPLOAD</param>
/// <param name="overWrite">DETERMINE IF THE FILE SHOULD BE OVERWRITTEN IF IT EXISTS</param>
/// <param name="extraInfo">EXTRA INFO TO STORE ABOUT THE FILE, WARNING: THESE PARAMETERS WILL BE STORED IN PLAIN TEXT</param>
/// <returns>RETURNS TRUE IF THE FILE WAS SUCCESSFULLY STORED OR FALSE IF IT FAILED</returns>
private bool DistributeFile(string folder, string fileName, int replicateCount, string tmpFileName, bool overWrite, ExtraInfo[] extraInfo)
{
List <string> successfulServers = new List <string>();
//ALL BELOW CHECKS ARE REALLY DONE TO MAKE SURE THE FILE DOES NOT EXISTS IN THE DATABASE ALREADY
//VERIFY THE FILENAME AND FOLDER ARE VALID
if (Regex.Match(fileName, @"[\/:*?""<>|]").Success || Regex.Match(folder, @"[:*?""<>|]").Success)
{
throw new ArgumentException("Invalid character(s) detected in filename and or folder");
}
//CHECK THE FORMAT OF THE FOLDER
folder = FormatFolder(folder);
//IF WE ARE NOT SUPPOSED TO OVERWRITE FILES THEN MAKE SURE IT DOES NOT ALREADY EXISTS IN THE FIRST DB IN THE LIST (IN THEORY IF IT DOES NOT HAVE IT NONE OF THEM SHOULD HAVE IT)
if (!overWrite)
{
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(ConfigurationManager.ConnectionStrings.Cast <ConnectionStringSettings>().ToArray().Shuffle()[0].ConnectionString))
{
//IF WE FIND A REFERENCE TO THIS FILE THEN WE NEED TO THROW AN ERROR
if ((from a in tmpDB.FileMarkers where a.FileName == fileName && a.Folder == folder select a).Count() != 0)
{
throw new Exception("File already exists");
}
}
}
else
{
//DELETE THE FILE MARKER FROM ALL THE DATABASE SERVERS IN THE CONNECTION STRING LIST
bool firstLoop = true;
foreach (ConnectionStringSettings tmpConnection in ConfigurationManager.ConnectionStrings)
{
//CREATE A NEW INSTANCE OF THE DATABASE ACCESS
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(tmpConnection.ConnectionString))
{
//DELETE ALL OF THE FILELOCATIONS FROM THE DATABASE
IEnumerable <FileMarker> tmpMarkers = tmpDB.FileMarkers.Where(a => a.Folder == folder && a.FileName == fileName);
//ATTEMPT TO DELETE THE REMOTE FILE TO CLEAR DISK SPACE
WebClient tmpClient = new WebClient();
if (tmpMarkers.Count() != 0 && firstLoop)
{
foreach (FileLocation tmpLocation in tmpMarkers.First().FileLocations)
{
try
{
tmpClient.DownloadData(string.Format("{0}DeleteFile.aspx?folder={1}&filename={2}", tmpLocation.Location, Server.UrlEncode(tmpLocation.FileMarker.Folder), Server.UrlEncode(tmpLocation.FileMarker.FileName)));
}
catch
{
//DO NOTHING IF WE ARE UNABLE TO DELETE THE FILE FOR ANY REASON
}
}
}
foreach (FileMarker tmpMarker in tmpMarkers)
{
tmpDB.FileLocations.DeleteAllOnSubmit(tmpMarker.FileLocations);
}
//DELETE ALL OF THE FILEMARKERS FROM THE DATABASE
tmpDB.FileMarkers.DeleteAllOnSubmit(tmpMarkers);
//SUBMIT THE CHANGES TO THE DATABASE
tmpDB.SubmitChanges();
//REMEMBER THIS IS NOT THE FIRST LOOP ANYMORE SO WE DO NOT TRY TO DELETE FILES AGAIN IF IN PMR STATE
firstLoop = false;
}
}
}
//NOW THE FUN PART TRY TO STORE THE FILE ON ANY SERVER WHICH WILL TAKE IT
try
{
//IF THE TEMP FOLDER DOES NOT EXISTS THEN CREATE IT SO WE CAN STORE THE FILE ON DISK AND UPLOAD IT TO THE REMOTE STORAGE LOCATION
if (!Directory.Exists(serverSettings.TempFolder))
{
Directory.CreateDirectory(serverSettings.TempFolder);
}
//USE WEBCLIENT FOR SIMPLE UPLOADS (THIS MAY NEED TO BE LOOKED INTO FOR DIRECT STREAM WRITING ON LARGE FILES)
WebClient tmpClient = new WebClient();
foreach (string chosenServer in (from a in serverSettings.RemoteStorage where a.AccessMode.Contains("w") select a.Path).ToArray().Shuffle())
{
try
{
string result = UploadFile(string.Format("{0}PutFile.aspx?folder={1}&filename={2}", chosenServer, Server.UrlEncode(folder), Server.UrlEncode(fileName)), tmpFileName);
//IF THE FILE UPLOAD WAS A SUCCESS THEN TRACK THE SERVER WE PLACED IT ON
if (result.ToLower() == "success")
{
successfulServers.Add(chosenServer);
//IF WE HAVE UPLOADED TO ENOUGH SERVERS TO SATISFY THE REPLICATION NEEDS THEN EXIT OUT AND CONTINUE ON
if (successfulServers.Count == replicateCount + 1)
{
break;
}
}
}
catch (Exception e)
{
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
}
}
catch (Exception e)
{
//HANDLE ERROR IF TEMP FOLDER DIES
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
//IF THE FILE WAS SAVED SUCCESSFULLY THEN RETURN TRUE TO THE USER AND SAVE THE MARKER FILE
if (successfulServers.Count == replicateCount + 1)
{
//CREATE A TEMPORARY MARKER TO BE STORED IN THE DATABASE
FileInfo tmpFileInfo = new FileInfo(tmpFileName);
FileMarker tmpMarker = new FileMarker();
tmpMarker.FileMarkerID = Guid.NewGuid().ToString();
tmpMarker.FileName = fileName;
tmpMarker.Folder = folder;
tmpMarker.Length = tmpFileInfo.Length;
tmpMarker.Hash = File.Open(tmpFileName, FileMode.Open).ComputeMD5(true);
tmpMarker.LastWriteTime = DateTime.Now;
tmpMarker.LastModTime = DateTime.Now;
tmpMarker.LastReadTime = DateTime.Now;
tmpMarker.ExtraInfo = extraInfo != null?System.Text.ASCIIEncoding.ASCII.GetBytes(extraInfo.Serialize().Compress()) : new byte[]
{
};
tmpMarker.FileLocations.AddRange(successfulServers.Select(a => new FileLocation()
{
Location = a, LocationID = Guid.NewGuid().ToString()
}).ToArray());
//WRITE THE FILE MARKER TO ALL THE DATABASE SERVERS IN THE CONNECTION STRING LIST
foreach (ConnectionStringSettings tmpConnection in ConfigurationManager.ConnectionStrings)
{
//CREATE A NEW INSTANCE OF THE DATABASE ACCESS
using (DistributedFileStorageDBDataContext tmpDB = new DistributedFileStorageDBDataContext(tmpConnection.ConnectionString))
{
//ADD THE FILE MARKER AND SUBMIT THE CHANGES
tmpDB.FileMarkers.InsertOnSubmit(tmpMarker);
tmpDB.SubmitChanges();
}
}
//RETURN THE SUCCESS TO THE CALLING FUNCTION
return(true);
}
else
{
//ATTEMPT TO NOW GO BACK AND DELETE ALL THE FILES WHICH DID SUCCESSFULLY UPLOAD
WebClient tmpClient = new WebClient();
foreach (string chosenServer in successfulServers)
{
try
{
string result = System.Text.ASCIIEncoding.ASCII.GetString(tmpClient.DownloadData(string.Format("{0}DeleteFile.aspx?folder={1}&filename={2}", chosenServer, Server.UrlEncode(folder), Server.UrlEncode(fileName))));
}
catch (Exception e)
{
Logging.WriteEntry(e.Message, e, new StackTrace(true));
}
}
//LET THE CALLING FUNCTION KNOW THE STORE FILE FAILED
return(false);
}
}
