/// <summary>
/// Fetches the files from Remote Storage for the restore script.
/// </summary>
/// <param name="database">The database.</param>
/// <param name="sortOrder">The sort order.</param>
/// <returns></returns>
public StringBuilder FetchFilesForRestoreScript(string database, ItemSortOrder sortOrder)
{
var remoteItems = m_remoteItemClass == ItemClass.Blob
? m_azureHelper.GetBlobItems()
: m_fileHelper.GetFileItems();
Console.WriteLine("Number of items found in Remote Storage: {0}", remoteItems.Count);
Console.WriteLine();
Console.WriteLine("Starting file review ...");
var filesToFetch = ParseLatestBackup(remoteItems, database);
var rfh = new RemoteFetchHelper(m_remoteItemClass);
// Actually fetch the files
var files = rfh.FetchItemsFromRemoteStorage(filesToFetch, sortOrder);
// Generate the script and return it
return(BuildRestoreScript(files));
}
public SyncHelper(ItemClass itemClass, bool debug)
{
m_remoteItemClass = itemClass;
m_common = new Common();
m_isDebug = debug;
if (m_remoteItemClass == ItemClass.Blob)
{
m_azureStorageHelper = m_azureStorageHelper ?? new AzureStorageHelper();
}
else
{
m_fileStorageHelper = m_fileStorageHelper ?? new FileStorageHelper();
m_remoteStorage = new DirectoryInfo(ConfigHelper.GetConfigurationValue("RemoteStorage"));
m_remoteUsername = ConfigHelper.GetFileStorageUsername();
m_remotePassword = ConfigHelper.GetFileStoragePassword();
}
m_isRunningOnWindows = Path.DirectorySeparatorChar.Equals('\\') ? true : false;
// Get the application settings
m_localDirectory = new DirectoryInfo(ConfigHelper.GetConfigurationValue("LocalPath"));
m_remoteStorage = new DirectoryInfo(ConfigHelper.GetConfigurationValue("RemoteStorage"));
m_copyFilesToRemoteStorage = ConfigHelper.GetConfigurationBoolean("CopyFilesToRemoteStorage");
m_deleteExplicitFilesFromRemoteStorage = ConfigHelper.GetConfigurationBoolean("DeleteExplicitFilesFromRemoteStorage");
m_fetchExplicitFilesFromRemoteStorage = ConfigHelper.GetConfigurationBoolean("FetchExplicitFilesFromRemoteStorage");
m_deleteMissingFilesFromRemoteStorage = ConfigHelper.GetConfigurationBoolean("DeleteMissingFilesFromRemoteStorage");
m_deleteMissingFilesFromLocalStorage = ConfigHelper.GetConfigurationBoolean("DeleteMissingFilesFromLocalStorage");
m_fetchFilesFromRemoteStorage = ConfigHelper.GetConfigurationBoolean("FetchFilesFromRemoteStorage");
m_explicitFilesToDeleteMatchingString = ConfigHelper.GetConfigurationValue("ExplicitFilesToDeleteMatchingString");
m_explicitFilesToFetchMatchingString = ConfigHelper.GetConfigurationValue("ExplicitFilesToFetchMatchingString");
m_itemSortOrder = ConfigHelper.GetConfigurationValue("ItemSortOrder")
.Equals("Size", StringComparison.InvariantCultureIgnoreCase)
? ItemSortOrder.Size
: ItemSortOrder.Name;
var compressAndEncrypt = ConfigHelper.GetConfigurationBoolean("CompressAndEncrypt");
var encryptOnly = ConfigHelper.GetConfigurationBoolean("EncryptOnly");
if (string.IsNullOrEmpty(m_explicitFilesToDeleteMatchingString))
{
// Don't delete any explicit files from Remote Path
m_deleteExplicitFilesFromRemoteStorage = false;
m_explicitFilesToDeleteMatchingString = string.Empty;
}
if (string.IsNullOrEmpty(m_explicitFilesToFetchMatchingString))
{
// Don't delete any explicit files from Remote Path
m_fetchExplicitFilesFromRemoteStorage = false;
m_explicitFilesToFetchMatchingString = string.Empty;
}
// Override the debug value
m_isDebug = !ConfigHelper.GetConfigurationValue("DebugOverride")
.Equals("True", StringComparison.InvariantCultureIgnoreCase);
// Encrypt and Compress settings -- only one can be true, default to EncryptOnly if set
if (compressAndEncrypt && encryptOnly)
{
compressAndEncrypt = false;
}
// Will only synchronise files that have the EncryptedFileExtension file type
// And this is only set if either of the encryption settings is set
m_syncEncryptedFilesOnly = compressAndEncrypt || encryptOnly;
m_encryptedFileExtension = ConfigHelper.GetConfigurationValue("EncryptedFileExtension") ?? ".absz";
}
/// <summary>
/// Generates the restore script.
/// </summary>
/// <param name="database">The database.</param>
/// <param name="dumpFileList">Dump list of all files on RemoteStorage</param>
/// <param name="sortOrder">The sort order</param>
/// <returns></returns>
public StringBuilder GenerateRestoreScript(string database, bool dumpFileList, ItemSortOrder sortOrder)
{
if (!dumpFileList)
{
return(FetchFilesForRestoreScript(database, sortOrder));
}
var remoteItems = m_remoteItemClass == ItemClass.Blob
? m_azureHelper.GetBlobItems()
: m_fileHelper.GetFileItems();
Console.WriteLine("Number of items found in Remote Storage: {0}", remoteItems.Count);
Console.WriteLine();
Console.WriteLine("Dumping list of Blob Items to disk ...");
var fileList = new StringBuilder();
foreach (var item in remoteItems)
{
fileList.AppendLine(item.Name);
}
return(fileList);
}
/// <summary>
/// Fetches the files from Remote Storage.
/// </summary>
/// <param name="filesToFetch">The files to fetch.</param>
/// <param name="sortOrder">The item sort order.</param>
/// <returns></returns>
public Dictionary <FileInfo, string> FetchItemsFromRemoteStorage(IEnumerable <RemoteItem> filesToFetch,
ItemSortOrder sortOrder)
{
var fileList = filesToFetch.ToList();
if (m_remoteItemClass == ItemClass.Blob)
{
var blobClient = m_azureHelper.StorageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(m_azureHelper.ContainerName);
const int segmentSize = 1 * 1024 * 1024; // 1 MB chunk
// int segmentSize = 64 * 1024; // 64 KB chunk
// Build list of files for restore script
var files = fileList.ToDictionary(file => new FileInfo($@"{m_localDirectory}\{file.Name.Replace(@"/", @"\")}"),
file => file.BackupType.ToFriendlyString());
// Sort by smallest file first
var sortedFilesToFetch = sortOrder == ItemSortOrder.Size
? fileList.OrderBy(x => x.Size)
: fileList.OrderBy(x => x.Name);
Console.WriteLine("Fetching files {0} with block size [{1}] ...",
sortOrder == ItemSortOrder.Size ? "from smallest to largest" : "in alphabetical order",
segmentSize.ToString("N0", CultureInfo.InvariantCulture));
foreach (var file in sortedFilesToFetch)
{
Console.Write("Fetching [{0}]: ", file.Name);
// Retrieve reference to a blob named "myblob".
var blockBlob = container.GetBlockBlobReference(file.Name);
var localFilePath = $@"{m_localDirectory}\{file.Name.Replace(@"/", @"\")}";
var fi = new FileInfo(localFilePath);
// Skip the file if it exists
if (fi.Exists && fi.Length == file.Size)
{
Common.SetFileCreationDate(file, fi);
Console.WriteLine("Done.");
continue;
}
if (fi.DirectoryName != null)
{
var di = new DirectoryInfo(fi.DirectoryName);
if (!di.Exists)
{
di.Create();
}
}
using (var fileStream = new FileStream(fi.FullName, FileMode.OpenOrCreate))
{
ParallelDownloadBlob(fileStream, blockBlob);
}
Common.SetFileCreationDate(file, fi);
Console.WriteLine("Done.");
}
return(files);
}
else
{
// Build list of files for restore script
var files = fileList.ToDictionary(file => new FileInfo($@"{file.PathOrUrl.Replace(m_remoteStorage, m_localDirectory)}\{file.Name}"),
file => file.BackupType.ToFriendlyString());
// Sort by smallest file first
var sortedFilesToFetch = sortOrder == ItemSortOrder.Size
? fileList.OrderBy(x => x.Size)
: fileList.OrderBy(x => x.Name);
Console.WriteLine("Fetching files {0} ...",
sortOrder == ItemSortOrder.Size ? "from smallest to largest" : "in alphabetical order");
foreach (var file in sortedFilesToFetch)
{
Console.Write("Fetching [{0}]: ", file.Name);
var remoteFilePath = $@"{file.PathOrUrl}\{file.Name}";
var localFilePath = remoteFilePath.Replace(m_remoteStorage, m_localDirectory);
var localFile = new FileInfo(localFilePath);
var remoteFile = new FileInfo(remoteFilePath);
// Skip the file if it exists
if (localFile.Exists && localFile.Length == file.Size)
{
Common.SetFileCreationDate(file, localFile);
Console.WriteLine("Done.");
continue;
}
if (localFile.DirectoryName != null)
{
var di = new DirectoryInfo(localFile.DirectoryName);
if (!di.Exists)
{
di.Create();
}
}
try
{
File.Copy(remoteFile.FullName, localFilePath);
Common.SetFileCreationDate(file, localFile);
Console.WriteLine("Done.");
}
catch (Exception)
{
Console.WriteLine("Failed copying file from {0} to {1}.", remoteFile.FullName, localFilePath);
}
}
return(files);
}
}
